[MEMORY] hold= 221,184 used= 214,432 count= 1 avg_used= 214,432 block_cnt= 1 chunk_cnt= 1 mod=CompSuggestMgr [MEMORY] hold= 212,992 used= 196,624 count= 2 avg_used= 98,312 block_cnt= 2 chunk_cnt= 1 mod=DRTaskMap [MEMORY] hold= 212,992 used= 196,624 count= 2 avg_used= 98,312 block_cnt= 2 chunk_cnt= 2 mod=DdlQue [MEMORY] hold= 212,992 used= 180,400 count= 4 avg_used= 45,100 block_cnt= 4 chunk_cnt= 4 mod=ProxySchemSQL [MEMORY] hold= 208,000 used= 206,336 count= 26 avg_used= 7,936 block_cnt= 26 chunk_cnt= 20 mod=ObSyncTabletSeq [MEMORY] hold= 196,608 used= 196,224 count= 3 avg_used= 65,408 block_cnt= 3 chunk_cnt= 2 mod=AsyncTaskQueue [MEMORY] hold= 194,480 used= 156,712 count= 11 avg_used= 14,246 block_cnt= 10 chunk_cnt= 7 mod=BucketLock [MEMORY] hold= 180,224 used= 172,064 count= 1 avg_used= 172,064 block_cnt= 1 chunk_cnt= 1 mod=TenantMBList [MEMORY] hold= 180,224 used= 135,784 count= 11 avg_used= 12,344 block_cnt= 11 chunk_cnt= 6 mod=ReferedMap [MEMORY] hold= 163,840 used= 147,792 count= 2 avg_used= 73,896 block_cnt= 2 chunk_cnt= 1 mod=HashNodNexWaiMa [MEMORY] hold= 163,504 used= 155,416 count= 2 avg_used= 77,708 block_cnt= 2 chunk_cnt= 2 mod=OB_DISK_REP [MEMORY] hold= 160,624 used= 159,272 count= 21 avg_used= 7,584 block_cnt= 21 chunk_cnt= 3 mod=TenaSpaTabIdSet [MEMORY] hold= 155,648 used= 147,624 count= 1 avg_used= 147,624 block_cnt= 1 chunk_cnt= 1 mod=UsrRuleMap [MEMORY] hold= 147,456 used= 110,736 count= 6 avg_used= 18,456 block_cnt= 6 chunk_cnt= 6 mod=HashBuckSysConf [MEMORY] hold= 144,880 used= 143,656 count= 19 avg_used= 7,560 block_cnt= 19 chunk_cnt= 4 mod=SysTableNameMap [MEMORY] hold= 138,240 used= 136,320 count= 30 avg_used= 4,544 block_cnt= 30 chunk_cnt= 13 mod=[T]ObTraceEvent [MEMORY] hold= 136,000 used= 134,912 count= 17 avg_used= 7,936 block_cnt= 17 chunk_cnt= 16 mod=SqlSessiVarMap [MEMORY] hold= 122,560 used= 118,904 count= 2 avg_used= 59,452 block_cnt= 2 chunk_cnt= 2 mod=TenaSchForCacMa [MEMORY] hold= 122,560 used= 118,904 count= 2 avg_used= 59,452 block_cnt= 2 chunk_cnt= 2 mod=RefrFullScheMap [MEMORY] hold= 122,560 used= 118,904 count= 2 avg_used= 59,452 block_cnt= 2 chunk_cnt= 2 mod=MemMgrMap [MEMORY] hold= 122,560 used= 118,904 count= 2 avg_used= 59,452 block_cnt= 2 chunk_cnt= 2 mod=MemMgrForLiboMa [MEMORY] hold= 114,688 used= 111,096 count= 1 avg_used= 111,096 block_cnt= 1 chunk_cnt= 1 mod=NonPartTenMap [MEMORY] hold= 114,688 used= 111,096 count= 1 avg_used= 111,096 block_cnt= 1 chunk_cnt= 1 mod=IndNameMap [MEMORY] hold= 114,336 used= 106,088 count= 2 avg_used= 53,044 block_cnt= 2 chunk_cnt= 2 mod=RetryCtrl [MEMORY] hold= 112,000 used= 111,104 count= 14 avg_used= 7,936 block_cnt= 14 chunk_cnt= 12 mod=ObDDLCommitClog [MEMORY] hold= 112,000 used= 111,104 count= 14 avg_used= 7,936 block_cnt= 14 chunk_cnt= 14 mod=ObDDLStartClogC [MEMORY] hold= 112,000 used= 111,104 count= 14 avg_used= 7,936 block_cnt= 14 chunk_cnt= 7 mod=HashMapArray [MEMORY] hold= 110,480 used= 105,224 count= 81 avg_used= 1,299 block_cnt= 41 chunk_cnt= 14 mod=ObGuard [MEMORY] hold= 106,496 used= 98,312 count= 1 avg_used= 98,312 block_cnt= 1 chunk_cnt= 1 mod=TmpFileManager [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 9 mod=PrepareOk [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 9 mod=acceptOK [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 9 mod=AddrList [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 9 mod=followerRenew [MEMORY] hold= 84,992 used= 55,968 count= 394 avg_used= 142 block_cnt= 2 chunk_cnt= 2 mod=TLDecoderCtx [MEMORY] hold= 82,432 used= 40,608 count= 542 avg_used= 74 block_cnt= 272 chunk_cnt= 27 mod=Coro [MEMORY] hold= 81,920 used= 46,960 count= 5 avg_used= 9,392 block_cnt= 5 chunk_cnt= 5 mod=TenCompProgMgr [MEMORY] hold= 80,000 used= 79,360 count= 10 avg_used= 7,936 block_cnt= 10 chunk_cnt= 9 mod=PlanVaIdx [MEMORY] hold= 73,728 used= 69,664 count= 1 avg_used= 69,664 block_cnt= 1 chunk_cnt= 1 mod=SuperBlockBuffe [MEMORY] hold= 73,728 used= 65,600 count= 1 avg_used= 65,600 block_cnt= 1 chunk_cnt= 1 mod=TCREF [MEMORY] hold= 64,816 used= 60,696 count= 4 avg_used= 15,174 block_cnt= 4 chunk_cnt= 3 mod=TmpFileStoreMap [MEMORY] hold= 64,000 used= 63,488 count= 8 avg_used= 7,936 block_cnt= 8 chunk_cnt= 8 mod=InneSqlConnPool [MEMORY] hold= 60,544 used= 51,200 count= 19 avg_used= 2,694 block_cnt= 15 chunk_cnt= 5 mod=TGTimer [MEMORY] hold= 58,736 used= 44,720 count= 215 avg_used= 208 block_cnt= 45 chunk_cnt= 10 mod=Scheduler [MEMORY] hold= 57,344 used= 55,368 count= 1 avg_used= 55,368 block_cnt= 1 chunk_cnt= 1 mod=DepInfoTaskQ [MEMORY] hold= 49,152 used= 32,768 count= 2 avg_used= 16,384 block_cnt= 2 chunk_cnt= 1 mod=CACHE_TNT_LST [MEMORY] hold= 48,832 used= 44,824 count= 2 avg_used= 22,412 block_cnt= 2 chunk_cnt= 2 mod=DDLSpeedCtrl [MEMORY] hold= 48,000 used= 47,616 count= 6 avg_used= 7,936 block_cnt= 6 chunk_cnt= 5 mod=MacroIdList [MEMORY] hold= 41,264 used= 37,264 count= 2 avg_used= 18,632 block_cnt= 2 chunk_cnt= 1 mod=TaskRunnerSer [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucPooUniMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucTenPooMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucSerUniMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucIdConfMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucNamPooMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HasBucSerMigUnM [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucNamConMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HasBucConRefCoM [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=ProxySessBuck [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SessHoldMapBuck [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucConPooMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucIdPoolMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=TableConnBucket [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=ObLongopsMgr [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SqlLoadData [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 5 mod=ObTsTenantInfoN [MEMORY] hold= 31,440 used= 7,344 count= 306 avg_used= 24 block_cnt= 153 chunk_cnt= 24 mod=[T]MemoryContex [MEMORY] hold= 30,464 used= 29,976 count= 7 avg_used= 4,282 block_cnt= 7 chunk_cnt= 6 mod=MemstoreAllocat [MEMORY] hold= 24,944 used= 20,784 count= 2 avg_used= 10,392 block_cnt= 2 chunk_cnt= 2 mod=SqlNio [MEMORY] hold= 24,672 used= 24,264 count= 6 avg_used= 4,044 block_cnt= 6 chunk_cnt= 3 mod=ScheObSchemAren [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=FuncRuleMap [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=leakMap [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=GrpIdNameMap [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=ServerCkptSlogH [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=GrpNameIdMap [MEMORY] hold= 24,256 used= 20,152 count= 2 avg_used= 10,076 block_cnt= 2 chunk_cnt= 2 mod=TbltRefreshMap [MEMORY] hold= 24,224 used= 21,672 count= 2 avg_used= 10,836 block_cnt= 2 chunk_cnt= 1 mod=SchemaStatuMap [MEMORY] hold= 24,000 used= 23,808 count= 3 avg_used= 7,936 block_cnt= 3 chunk_cnt= 2 mod=ObBlackListHash [MEMORY] hold= 23,616 used= 23,424 count= 3 avg_used= 7,808 block_cnt= 3 chunk_cnt= 3 mod=ProxySessNode [MEMORY] hold= 23,520 used= 23,160 count= 5 avg_used= 4,632 block_cnt= 5 chunk_cnt= 5 mod=SqlPx [MEMORY] hold= 23,520 used= 23,160 count= 5 avg_used= 4,632 block_cnt= 5 chunk_cnt= 5 mod=RebuildCtx [MEMORY] hold= 21,504 used= 20,352 count= 16 avg_used= 1,272 block_cnt= 16 chunk_cnt= 2 mod=IO_GROUP_MAP [MEMORY] hold= 18,832 used= 14,040 count= 65 avg_used= 216 block_cnt= 39 chunk_cnt= 31 mod=ObFuture [MEMORY] hold= 18,544 used= 17,408 count= 17 avg_used= 1,024 block_cnt= 12 chunk_cnt= 9 mod=timer [MEMORY] hold= 18,416 used= 18,144 count= 4 avg_used= 4,536 block_cnt= 4 chunk_cnt= 2 mod=DeviceMng [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=HasBucTimZonInM [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=StorageHADiag [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=ServerLogPool [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 8,992 count= 1 avg_used= 8,992 block_cnt= 1 chunk_cnt= 1 mod=IO_HEALTH [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=MemDumpMap [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=LinkArray [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=GenSchemVersMap [MEMORY] hold= 16,384 used= 9,336 count= 1 avg_used= 9,336 block_cnt= 1 chunk_cnt= 1 mod=InnerLobHash [MEMORY] hold= 16,384 used= 12,296 count= 1 avg_used= 12,296 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 16,320 used= 16,000 count= 5 avg_used= 3,200 block_cnt= 5 chunk_cnt= 5 mod=TenantTZ [MEMORY] hold= 16,032 used= 15,904 count= 2 avg_used= 7,952 block_cnt= 2 chunk_cnt= 1 mod=UpgProcSet [MEMORY] hold= 16,000 used= 15,872 count= 2 avg_used= 7,936 block_cnt= 2 chunk_cnt= 1 mod=CommSysVarDefVa [MEMORY] hold= 16,000 used= 15,872 count= 2 avg_used= 7,936 block_cnt= 2 chunk_cnt= 2 mod=DisplayList [MEMORY] hold= 15,744 used= 15,616 count= 2 avg_used= 7,808 block_cnt= 2 chunk_cnt= 2 mod=SessHoldMapNode [MEMORY] hold= 14,288 used= 8,352 count= 87 avg_used= 96 block_cnt= 39 chunk_cnt= 7 mod=PThread [MEMORY] hold= 10,240 used= 9,920 count= 5 avg_used= 1,984 block_cnt= 5 chunk_cnt= 5 mod=LobManager [MEMORY] hold= 8,288 used= 8,048 count= 3 avg_used= 2,682 block_cnt= 3 chunk_cnt= 3 mod=TenantInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=SchemaRowKey [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=TbltRefTasks [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DDLClogCtrl [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ObBLValue [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=lock_diagnose [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DutyTask [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=TbltReSrv [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=RetryTask [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ServerBlacklist [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ServerCidMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ServerRegioMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ServerIdcMap [MEMORY] hold= 7,968 used= 7,904 count= 1 avg_used= 7,904 block_cnt= 1 chunk_cnt= 1 mod=LuaHandler [MEMORY] hold= 7,872 used= 7,808 count= 1 avg_used= 7,808 block_cnt= 1 chunk_cnt= 1 mod=HasNodTzInfM [MEMORY] hold= 7,808 used= 6,528 count= 16 avg_used= 408 block_cnt= 16 chunk_cnt= 15 mod=[T]ObTLDecoderC [MEMORY] hold= 7,360 used= 7,000 count= 5 avg_used= 1,400 block_cnt= 5 chunk_cnt= 5 mod=GtiRequestRpc [MEMORY] hold= 7,040 used= 6,889 count= 2 avg_used= 3,444 block_cnt= 2 chunk_cnt= 2 mod=RpcProcessor [MEMORY] hold= 6,960 used= 6,640 count= 5 avg_used= 1,328 block_cnt= 5 chunk_cnt= 5 mod=GtiRpcProxy [MEMORY] hold= 5,376 used= 5,248 count= 2 avg_used= 2,624 block_cnt= 1 chunk_cnt= 1 mod=RootContext [MEMORY] hold= 5,216 used= 4,608 count= 9 avg_used= 512 block_cnt= 8 chunk_cnt= 8 mod=SqlString [MEMORY] hold= 5,184 used= 4,608 count= 9 avg_used= 512 block_cnt= 7 chunk_cnt= 7 mod=SqlSessiQuerSql [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmReq [MEMORY] hold= 4,480 used= 3,816 count= 9 avg_used= 424 block_cnt= 7 chunk_cnt= 7 mod=SequenceMap [MEMORY] hold= 4,464 used= 3,816 count= 9 avg_used= 424 block_cnt= 7 chunk_cnt= 7 mod=ContextsMap [MEMORY] hold= 4,464 used= 3,816 count= 9 avg_used= 424 block_cnt= 7 chunk_cnt= 7 mod=SequenceIdMap [MEMORY] hold= 4,464 used= 4,032 count= 6 avg_used= 672 block_cnt= 6 chunk_cnt= 6 mod=[T]ObWarningBuf [MEMORY] hold= 4,464 used= 3,816 count= 9 avg_used= 424 block_cnt= 7 chunk_cnt= 7 mod=PackStateMap [MEMORY] hold= 3,888 used= 3,816 count= 1 avg_used= 3,816 block_cnt= 1 chunk_cnt= 1 mod=RemMasterMap [MEMORY] hold= 3,888 used= 3,816 count= 1 avg_used= 3,816 block_cnt= 1 chunk_cnt= 1 mod=RecScheHisMap [MEMORY] hold= 3,840 used= 3,520 count= 5 avg_used= 704 block_cnt= 3 chunk_cnt= 3 mod=ScheMgrCacheMap [MEMORY] hold= 3,312 used= 2,880 count= 6 avg_used= 480 block_cnt= 6 chunk_cnt= 6 mod=TntResourceMgr [MEMORY] hold= 3,136 used= 3,072 count= 1 avg_used= 3,072 block_cnt= 1 chunk_cnt= 1 mod=TsMgr [MEMORY] hold= 2,768 used= 2,704 count= 1 avg_used= 2,704 block_cnt= 1 chunk_cnt= 1 mod=LoggerAlloc [MEMORY] hold= 1,760 used= 880 count= 10 avg_used= 88 block_cnt= 10 chunk_cnt= 5 mod=DestKAState [MEMORY] hold= 1,504 used= 1,432 count= 1 avg_used= 1,432 block_cnt= 1 chunk_cnt= 1 mod=GtsRequestRpc [MEMORY] hold= 1,408 used= 1,344 count= 1 avg_used= 1,344 block_cnt= 1 chunk_cnt= 1 mod=SchemaService [MEMORY] hold= 1,392 used= 1,328 count= 1 avg_used= 1,328 block_cnt= 1 chunk_cnt= 1 mod=GtsRpcProxy [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=IO_TENANT_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=INGRESS_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=IO_CHANNEL_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=TENANT_PLAN_MAP [MEMORY] hold= 1,152 used= 1,088 count= 1 avg_used= 1,088 block_cnt= 1 chunk_cnt= 1 mod=memdumpqueue [MEMORY] hold= 960 used= 280 count= 10 avg_used= 28 block_cnt= 3 chunk_cnt= 3 mod=KeepAliveServer [MEMORY] hold= 800 used= 336 count= 7 avg_used= 48 block_cnt= 7 chunk_cnt= 6 mod=[T]TSITabletTab [MEMORY] hold= 752 used= 336 count= 6 avg_used= 56 block_cnt= 6 chunk_cnt= 5 mod=UserResourceMgr [MEMORY] hold= 640 used= 280 count= 5 avg_used= 56 block_cnt= 5 chunk_cnt= 4 mod=PxTargetMgr [MEMORY] hold= 640 used= 280 count= 5 avg_used= 56 block_cnt= 5 chunk_cnt= 1 mod=Log [MEMORY] hold= 560 used= 14 count= 7 avg_used= 2 block_cnt= 6 chunk_cnt= 6 mod=[T]TSIUseWeak [MEMORY] hold= 480 used= 96 count= 6 avg_used= 16 block_cnt= 5 chunk_cnt= 5 mod=CreateEntity [MEMORY] hold= 416 used= 272 count= 2 avg_used= 136 block_cnt= 1 chunk_cnt= 1 mod=unknown [MEMORY] hold= 288 used= 216 count= 1 avg_used= 216 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 224 used= 160 count= 1 avg_used= 160 block_cnt= 1 chunk_cnt= 1 mod=OccamTimeGuard [MEMORY] hold= 192 used= 48 count= 2 avg_used= 24 block_cnt= 2 chunk_cnt= 2 mod=RemoteTask [MEMORY] hold= 80 used= 7 count= 1 avg_used= 7 block_cnt= 1 chunk_cnt= 1 mod=SqlExpr [MEMORY] hold= 318,519,120 used= 309,351,429 count= 56,912 avg_used= 5,435 mod=SUMMARY [2024-06-25 17:00:18.759123] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=34] [MEMORY] tenant_id= 500 ctx_id= GLIBC hold= 20,971,520 used= 13,250,080 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 12,935,968 used= 8,521,245 count= 60,966 avg_used= 139 block_cnt= 1,571 chunk_cnt= 10 mod=glibc_malloc [MEMORY] hold= 311,456 used= 89,560 count= 2,942 avg_used= 30 block_cnt= 42 chunk_cnt= 1 mod=Buffer [MEMORY] hold= 2,656 used= 1,222 count= 20 avg_used= 61 block_cnt= 5 chunk_cnt= 1 mod=XmlGlobal [MEMORY] hold= 13,250,080 used= 8,612,027 count= 63,928 avg_used= 134 mod=SUMMARY [2024-06-25 17:00:18.759130] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 500 ctx_id= CO_STACK hold= 165,675,008 used= 163,086,336 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 163,086,336 used= 162,782,976 count= 316 avg_used= 515,136 block_cnt= 316 chunk_cnt= 79 mod=CoStack [MEMORY] hold= 163,086,336 used= 162,782,976 count= 316 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.759138] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 500 ctx_id= LIBEASY hold= 8,388,608 used= 4,223,872 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 4,223,872 used= 3,403,736 count= 599 avg_used= 5,682 block_cnt= 136 chunk_cnt= 4 mod=OB_TEST2_PCODE [MEMORY] hold= 4,223,872 used= 3,403,736 count= 599 avg_used= 5,682 mod=SUMMARY [2024-06-25 17:00:18.759146] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 500 ctx_id= LOGGER_CTX_ID hold= 14,680,064 used= 14,565,376 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 10,403,840 used= 10,398,720 count= 5 avg_used= 2,079,744 block_cnt= 5 chunk_cnt= 5 mod=Logger [MEMORY] hold= 4,161,536 used= 4,159,488 count= 2 avg_used= 2,079,744 block_cnt= 2 chunk_cnt= 2 mod=ErrorLogger [MEMORY] hold= 14,565,376 used= 14,558,208 count= 7 avg_used= 2,079,744 mod=SUMMARY [2024-06-25 17:00:18.759165] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 500 ctx_id= RPC_CTX_ID hold= 10,485,760 used= 1,294,336 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 1,277,952 used= 1,267,968 count= 78 avg_used= 16,256 block_cnt= 78 chunk_cnt= 5 mod=RpcDefault [MEMORY] hold= 16,384 used= 16,256 count= 1 avg_used= 16,256 block_cnt= 1 chunk_cnt= 1 mod=OB_PX_TARGET_RE [MEMORY] hold= 1,294,336 used= 1,284,224 count= 79 avg_used= 16,256 mod=SUMMARY [2024-06-25 17:00:18.759177] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 500 ctx_id= PKT_NIO hold= 160,014,336 used= 122,969,280 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 70,679,744 used= 70,248,480 count= 384 avg_used= 182,938 block_cnt= 143 chunk_cnt= 40 mod=DEFAULT [MEMORY] hold= 23,658,496 used= 23,575,680 count= 180 avg_used= 130,976 block_cnt= 180 chunk_cnt= 29 mod=PKTC_INBUF [MEMORY] hold= 23,617,536 used= 23,575,680 count= 180 avg_used= 130,976 block_cnt= 180 chunk_cnt= 29 mod=PKTS_INBUF [MEMORY] hold= 1,474,560 used= 1,465,920 count= 90 avg_used= 16,288 block_cnt= 90 chunk_cnt= 30 mod=SERVER_CTX_CHUN [MEMORY] hold= 1,179,648 used= 1,172,736 count= 72 avg_used= 16,288 block_cnt= 72 chunk_cnt= 29 mod=CLIENT_CB_CHUNK [MEMORY] hold= 1,179,648 used= 1,172,736 count= 72 avg_used= 16,288 block_cnt= 72 chunk_cnt= 15 mod=CLIENT_REQ_CHUN [MEMORY] hold= 1,179,648 used= 1,172,736 count= 72 avg_used= 16,288 block_cnt= 72 chunk_cnt= 28 mod=SERVER_RESP_CHU [MEMORY] hold= 122,969,280 used= 122,383,968 count= 1,050 avg_used= 116,556 mod=SUMMARY [2024-06-25 17:00:18.759228] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 500 ctx_id= SCHEMA_SERVICE hold= 158,093,312 used= 139,296,224 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 68,745,344 used= 68,710,912 count= 43 avg_used= 1,597,928 block_cnt= 43 chunk_cnt= 40 mod=TenantSchemMgr [MEMORY] hold= 60,561,856 used= 60,052,014 count= 6,745 avg_used= 8,903 block_cnt= 1,027 chunk_cnt= 38 mod=SchemaSysCache [MEMORY] hold= 7,129,472 used= 7,109,672 count= 5 avg_used= 1,421,934 block_cnt= 5 chunk_cnt= 2 mod=SchemaIdVersion [MEMORY] hold= 884,736 used= 786,432 count= 12 avg_used= 65,536 block_cnt= 12 chunk_cnt= 4 mod=SchemaMgrCache [MEMORY] hold= 495,360 used= 380,392 count= 36 avg_used= 10,566 block_cnt= 36 chunk_cnt= 14 mod=ScheTablInfoVec [MEMORY] hold= 208,000 used= 206,336 count= 26 avg_used= 7,936 block_cnt= 26 chunk_cnt= 12 mod=ScheTablIdMap [MEMORY] hold= 159,360 used= 158,080 count= 20 avg_used= 7,904 block_cnt= 20 chunk_cnt= 10 mod=ScheTenaInfoVec [MEMORY] hold= 144,000 used= 142,848 count= 18 avg_used= 7,936 block_cnt= 18 chunk_cnt= 8 mod=ScheTablNameMap [MEMORY] hold= 137,024 used= 114,392 count= 13 avg_used= 8,799 block_cnt= 13 chunk_cnt= 6 mod=ScheIndeInfoVec [MEMORY] hold= 87,648 used= 86,944 count= 11 avg_used= 7,904 block_cnt= 11 chunk_cnt= 8 mod=ScheLobPInfoVec [MEMORY] hold= 87,648 used= 86,944 count= 11 avg_used= 7,904 block_cnt= 11 chunk_cnt= 5 mod=ScheLobMInfoVec [MEMORY] hold= 81,664 used= 80,320 count= 20 avg_used= 4,016 block_cnt= 19 chunk_cnt= 12 mod=SchemaKeystore [MEMORY] hold= 81,664 used= 80,320 count= 20 avg_used= 4,016 block_cnt= 20 chunk_cnt= 10 mod=SchemaSysVariab [MEMORY] hold= 79,680 used= 79,040 count= 10 avg_used= 7,904 block_cnt= 10 chunk_cnt= 7 mod=SchePrivDbPrivs [MEMORY] hold= 79,680 used= 79,040 count= 10 avg_used= 7,904 block_cnt= 10 chunk_cnt= 7 mod=ScheDbInfoVec [MEMORY] hold= 72,288 used= 71,712 count= 9 avg_used= 7,968 block_cnt= 9 chunk_cnt= 4 mod=ScheMgrInfoArra [MEMORY] hold= 48,896 used= 48,128 count= 12 avg_used= 4,010 block_cnt= 12 chunk_cnt= 5 mod=ScheIndeNameMap [MEMORY] hold= 17,888 used= 17,088 count= 12 avg_used= 1,424 block_cnt= 12 chunk_cnt= 8 mod=SchemaSequence [MEMORY] hold= 16,064 used= 15,808 count= 2 avg_used= 7,904 block_cnt= 2 chunk_cnt= 2 mod=SchePackInfoVec [MEMORY] hold= 16,064 used= 15,808 count= 2 avg_used= 7,904 block_cnt= 2 chunk_cnt= 2 mod=ScheRoutInfoVec [MEMORY] hold= 10,880 used= 10,240 count= 10 avg_used= 1,024 block_cnt= 10 chunk_cnt= 6 mod=ScheRoutNameMap [MEMORY] hold= 10,880 used= 10,240 count= 10 avg_used= 1,024 block_cnt= 10 chunk_cnt= 6 mod=SchePackIdMap [MEMORY] hold= 10,880 used= 10,240 count= 10 avg_used= 1,024 block_cnt= 9 chunk_cnt= 5 mod=SchePackNameMap [MEMORY] hold= 10,880 used= 10,240 count= 10 avg_used= 1,024 block_cnt= 10 chunk_cnt= 6 mod=ScheRoutIdMap [MEMORY] hold= 10,880 used= 10,240 count= 10 avg_used= 1,024 block_cnt= 10 chunk_cnt= 5 mod=HiddenTblNames [MEMORY] hold= 8,288 used= 5,120 count= 40 avg_used= 128 block_cnt= 25 chunk_cnt= 11 mod=ScheLabeSeCompo [MEMORY] hold= 6,240 used= 3,840 count= 30 avg_used= 128 block_cnt= 20 chunk_cnt= 9 mod=ScheLabeSePolic [MEMORY] hold= 6,176 used= 3,840 count= 30 avg_used= 128 block_cnt= 23 chunk_cnt= 10 mod=ScheLabeSeLabel [MEMORY] hold= 5,760 used= 5,120 count= 10 avg_used= 512 block_cnt= 10 chunk_cnt= 6 mod=ScheUdtNameMap [MEMORY] hold= 5,760 used= 5,120 count= 10 avg_used= 512 block_cnt= 10 chunk_cnt= 6 mod=ScheUdtIdMap [MEMORY] hold= 4,096 used= 2,560 count= 20 avg_used= 128 block_cnt= 17 chunk_cnt= 8 mod=ScheLabSeUserLe [MEMORY] hold= 4,032 used= 2,560 count= 20 avg_used= 128 block_cnt= 13 chunk_cnt= 7 mod=RLS_CONTEXT_MGR [MEMORY] hold= 4,032 used= 2,560 count= 20 avg_used= 128 block_cnt= 13 chunk_cnt= 6 mod=RLS_GROUP_MGR [MEMORY] hold= 4,032 used= 2,560 count= 20 avg_used= 128 block_cnt= 13 chunk_cnt= 6 mod=RLS_POLICY_MGR [MEMORY] hold= 4,000 used= 2,560 count= 20 avg_used= 128 block_cnt= 15 chunk_cnt= 8 mod=DIRECTORY_MGR [MEMORY] hold= 3,904 used= 2,560 count= 20 avg_used= 128 block_cnt= 11 chunk_cnt= 7 mod=SchemaProfile [MEMORY] hold= 3,904 used= 2,560 count= 20 avg_used= 128 block_cnt= 12 chunk_cnt= 6 mod=SchemaSynonym [MEMORY] hold= 3,904 used= 2,560 count= 20 avg_used= 128 block_cnt= 13 chunk_cnt= 8 mod=DBLINK_MGR [MEMORY] hold= 3,840 used= 2,560 count= 20 avg_used= 128 block_cnt= 12 chunk_cnt= 9 mod=ScheOutlSqlMap [MEMORY] hold= 3,712 used= 3,072 count= 10 avg_used= 307 block_cnt= 9 chunk_cnt= 6 mod=ScheDataNameMap [MEMORY] hold= 3,280 used= 2,920 count= 5 avg_used= 584 block_cnt= 5 chunk_cnt= 4 mod=TenaScheMemMgr [MEMORY] hold= 3,280 used= 2,920 count= 5 avg_used= 584 block_cnt= 4 chunk_cnt= 3 mod=TenSchMemMgrFoL [MEMORY] hold= 2,080 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 7 mod=SchemaTablespac [MEMORY] hold= 1,984 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=SchemaUdf [MEMORY] hold= 1,984 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=SchemaSecurAudi [MEMORY] hold= 1,952 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=PRIV_ROUTINE [MEMORY] hold= 1,952 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 7 mod=ScheConsNameMap [MEMORY] hold= 1,952 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 7 mod=ScheForKeyNamMa [MEMORY] hold= 1,952 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 8 mod=ScheAuxVpNameVe [MEMORY] hold= 1,952 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=ScheTrigNameMap [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=SchePriTabPriMa [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 9 chunk_cnt= 6 mod=SchePriObjPriMa [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 9 chunk_cnt= 4 mod=ScheTrigIdMap [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 9 chunk_cnt= 7 mod=ScheOutlNameMap [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 7 mod=ScheOutlIdMap [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=MockFkParentTab [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 5 mod=SchemaContext [MEMORY] hold= 139,296,224 used= 138,447,622 count= 7,557 avg_used= 18,320 mod=SUMMARY [2024-06-25 17:00:18.759254] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=16] [MEMORY] tenant_id= 500 ctx_id= UNEXPECTED_IN_500 hold= 2,473,402,368 used= 2,440,823,872 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 1,745,096,880 used= 1,744,830,568 count= 14 avg_used= 124,630,754 block_cnt= 14 chunk_cnt= 14 mod=CACHE_MAP_BKT [MEMORY] hold= 268,546,944 used= 268,444,576 count= 7 avg_used= 38,349,225 block_cnt= 7 chunk_cnt= 6 mod=FixeSizeBlocAll [MEMORY] hold= 201,347,072 used= 201,326,616 count= 1 avg_used= 201,326,616 block_cnt= 1 chunk_cnt= 1 mod=CACHE_MAP_LOCK [MEMORY] hold= 78,606,336 used= 78,587,160 count= 1 avg_used= 78,587,160 block_cnt= 1 chunk_cnt= 1 mod=CACHE_MB_HANDLE [MEMORY] hold= 64,519,168 used= 64,471,552 count= 292 avg_used= 220,792 block_cnt= 292 chunk_cnt= 57 mod=BlockMap [MEMORY] hold= 34,734,080 used= 34,611,840 count= 40 avg_used= 865,296 block_cnt= 40 chunk_cnt= 23 mod=di_tenant_cache [MEMORY] hold= 17,329,216 used= 8,968,960 count= 1,093 avg_used= 8,205 block_cnt= 1,093 chunk_cnt= 9 mod=StorageLoggerM [MEMORY] hold= 15,550,048 used= 13,817,664 count= 398 avg_used= 34,717 block_cnt= 230 chunk_cnt= 11 mod=OccamThreadPool [MEMORY] hold= 12,042,240 used= 11,947,880 count= 5 avg_used= 2,389,576 block_cnt= 5 chunk_cnt= 5 mod=TenantConfig [MEMORY] hold= 3,036,992 used= 2,917,447 count= 157 avg_used= 18,582 block_cnt= 126 chunk_cnt= 23 mod=CommonNetwork [MEMORY] hold= 14,896 used= 2,656 count= 166 avg_used= 16 block_cnt= 16 chunk_cnt= 3 mod=ConfigChecker [MEMORY] hold= 2,440,823,872 used= 2,429,926,919 count= 2,174 avg_used= 1,117,721 mod=SUMMARY [2024-06-25 17:00:18.759268] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=6] [MEMORY] tenant: 508, limit: 1,073,741,824 hold: 40,296,448 rpc_hold: 0 cache_hold: 0 cache_used: 0 cache_item_count: 0 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 13,033,472 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 27,262,976 limit= 9,223,372,036,854,775,807 [2024-06-25 17:00:18.759284] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 508 ctx_id= DEFAULT_CTX_ID hold= 13,033,472 used= 11,676,016 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 3,833,856 used= 3,423,264 count= 52 avg_used= 65,832 block_cnt= 52 chunk_cnt= 3 mod=[T]ObSessionDIB [MEMORY] hold= 3,760,128 used= 3,342,336 count= 51 avg_used= 65,536 block_cnt= 51 chunk_cnt= 4 mod=[T]char [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 413,712 used= 410,448 count= 51 avg_used= 8,048 block_cnt= 51 chunk_cnt= 4 mod=[T]ObPerfEventR [MEMORY] hold= 235,008 used= 231,744 count= 51 avg_used= 4,544 block_cnt= 51 chunk_cnt= 4 mod=[T]ObTraceEvent [MEMORY] hold= 98,304 used= 94,848 count= 1 avg_used= 94,848 block_cnt= 1 chunk_cnt= 1 mod=ResourceGroup [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DynamicFactor [MEMORY] hold= 37,536 used= 34,272 count= 51 avg_used= 672 block_cnt= 51 chunk_cnt= 4 mod=[T]ObWarningBuf [MEMORY] hold= 24,960 used= 21,632 count= 52 avg_used= 416 block_cnt= 7 chunk_cnt= 2 mod=OMT_Worker [MEMORY] hold= 14,144 used= 6,656 count= 104 avg_used= 64 block_cnt= 7 chunk_cnt= 2 mod=Coro [MEMORY] hold= 4,992 used= 1,248 count= 52 avg_used= 24 block_cnt= 7 chunk_cnt= 2 mod=[T]MemoryContex [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 11,676,016 used= 10,793,616 count= 469 avg_used= 23,014 mod=SUMMARY [2024-06-25 17:00:18.759305] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=6] [MEMORY] tenant_id= 508 ctx_id= CO_STACK hold= 27,262,976 used= 26,836,992 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 26,836,992 used= 26,787,072 count= 52 avg_used= 515,136 block_cnt= 52 chunk_cnt= 13 mod=CoStack [MEMORY] hold= 26,836,992 used= 26,787,072 count= 52 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.759391] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant: 1, limit: 40,399,536,128 hold: 17,737,887,744 rpc_hold: 0 cache_hold: 14,854,127,616 cache_used: 14,854,127,616 cache_item_count: 7,083 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 2,015,518,720 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MEMSTORE_CTX_ID hold_bytes= 285,212,672 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= TRANS_CTX_MGR_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PLAN_CACHE_CTX_ID hold_bytes= 312,475,648 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= GLIBC hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 146,800,640 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= META_OBJ_CTX_ID hold_bytes= 20,971,520 limit= 8,079,907,220 [MEMORY] ctx_id= TX_CALLBACK_CTX_ID hold_bytes= 20,971,520 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= LOB_CTX_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PS_CACHE_CTX_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= TX_DATA_TABLE hold_bytes= 69,226,496 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MDS_DATA_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY][PM] tid= 62760 used= 0 hold= 4,194,304 pm=0x2b3a5aad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62761 used= 0 hold= 2,097,152 pm=0x2b3a5ab52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62764 used= 0 hold= 4,194,304 pm=0x2b3a5acd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62769 used= 0 hold= 4,194,304 pm=0x2b3a5b152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62770 used= 0 hold= 4,194,304 pm=0x2b3a5b1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62771 used= 0 hold= 4,194,304 pm=0x2b3a5b256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62772 used= 0 hold= 4,194,304 pm=0x2b3a5b2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62773 used= 0 hold= 4,194,304 pm=0x2b3a5b352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62774 used= 0 hold= 4,194,304 pm=0x2b3a5b3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62775 used= 0 hold= 4,194,304 pm=0x2b3a5b656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62776 used= 0 hold= 4,194,304 pm=0x2b3a5b6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62777 used= 0 hold= 4,194,304 pm=0x2b3a5b752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62778 used= 0 hold= 4,194,304 pm=0x2b3a5b7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62779 used= 0 hold= 4,194,304 pm=0x2b3a5b856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62780 used= 0 hold= 4,194,304 pm=0x2b3a5b8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62781 used= 0 hold= 4,194,304 pm=0x2b3a5b952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62782 used= 0 hold= 4,194,304 pm=0x2b3a5b9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62783 used= 0 hold= 4,194,304 pm=0x2b3a5ba56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62784 used= 0 hold= 4,194,304 pm=0x2b3a5bad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62791 used= 0 hold= 2,097,152 pm=0x2b3a5bb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62792 used= 0 hold= 2,097,152 pm=0x2b3a5bbd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62793 used= 0 hold= 2,097,152 pm=0x2b3a5c656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62794 used= 0 hold= 2,097,152 pm=0x2b3a5c6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62913 used= 0 hold= 4,194,304 pm=0x2b3a66fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62914 used= 0 hold= 4,194,304 pm=0x2b3a7e856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62915 used= 0 hold= 4,194,304 pm=0x2b3a7e8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62916 used= 0 hold= 4,194,304 pm=0x2b3a7e952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62917 used= 0 hold= 4,194,304 pm=0x2b3a7e9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62918 used= 0 hold= 4,194,304 pm=0x2b3a7ea56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62919 used= 0 hold= 4,194,304 pm=0x2b3a7ead4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62920 used= 0 hold= 4,194,304 pm=0x2b3a7eb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62921 used= 0 hold= 4,194,304 pm=0x2b3a7ebd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62922 used= 0 hold= 4,194,304 pm=0x2b3a7ec56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62923 used= 0 hold= 4,194,304 pm=0x2b3a7ecd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62924 used= 0 hold= 4,194,304 pm=0x2b3a7ed52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62925 used= 0 hold= 4,194,304 pm=0x2b3a7edd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62926 used= 0 hold= 4,194,304 pm=0x2b3a7ee56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62927 used= 0 hold= 4,194,304 pm=0x2b3a7eed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62928 used= 0 hold= 4,194,304 pm=0x2b3a7ef52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62929 used= 2,145,344 hold= 4,194,304 pm=0x2b3a7efd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62930 used= 0 hold= 4,194,304 pm=0x2b3a7f256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62931 used= 0 hold= 4,194,304 pm=0x2b3a7f2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62932 used= 0 hold= 4,194,304 pm=0x2b3a7f352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62933 used= 0 hold= 4,194,304 pm=0x2b3a7f3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62934 used= 0 hold= 4,194,304 pm=0x2b3a7f456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62935 used= 0 hold= 4,194,304 pm=0x2b3a7f4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62936 used= 0 hold= 4,194,304 pm=0x2b3a7f552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77744 used= 0 hold= 2,097,152 pm=0x2bd04c1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77745 used= 0 hold= 2,097,152 pm=0x2bd04c256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77746 used= 0 hold= 4,194,304 pm=0x2bd04c2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77747 used= 0 hold= 4,194,304 pm=0x2bd04c352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77748 used= 0 hold= 4,194,304 pm=0x2bd04c3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77749 used= 0 hold= 4,194,304 pm=0x2bd04c456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77750 used= 0 hold= 4,194,304 pm=0x2bd04c4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77751 used= 0 hold= 4,194,304 pm=0x2bd04c552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77752 used= 0 hold= 4,194,304 pm=0x2bd04c5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77753 used= 0 hold= 4,194,304 pm=0x2bd04c656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77754 used= 0 hold= 4,194,304 pm=0x2bd04c6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77755 used= 0 hold= 4,194,304 pm=0x2bd04c752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77756 used= 0 hold= 4,194,304 pm=0x2bd04c7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77757 used= 0 hold= 4,194,304 pm=0x2bd04c856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77758 used= 0 hold= 4,194,304 pm=0x2bd04c8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77759 used= 0 hold= 4,194,304 pm=0x2bd04c952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77760 used= 0 hold= 4,194,304 pm=0x2bd04c9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77761 used= 0 hold= 4,194,304 pm=0x2bd050456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77762 used= 0 hold= 4,194,304 pm=0x2bd0504d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77763 used= 0 hold= 4,194,304 pm=0x2bd050552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77764 used= 0 hold= 4,194,304 pm=0x2bd0505d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77765 used= 0 hold= 4,194,304 pm=0x2bd050656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= summary used= 2,145,344 hold= 274,726,912 [2024-06-25 17:00:18.759543] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=11] [MEMORY] tenant_id= 1 ctx_id= DEFAULT_CTX_ID hold= 2,015,518,720 used= 1,617,364,480 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 847,839,232 used= 847,436,480 count= 371 avg_used= 2,284,195 block_cnt= 371 chunk_cnt= 371 mod=MysqlRequesReco [MEMORY] hold= 132,341,760 used= 117,637,120 count= 1,795 avg_used= 65,536 block_cnt= 1,795 chunk_cnt= 177 mod=MinorMergeMgr [MEMORY] hold= 76,988,416 used= 76,950,528 count= 37 avg_used= 2,079,744 block_cnt= 37 chunk_cnt= 37 mod=CACHE_MAP_NODE [MEMORY] hold= 76,742,656 used= 76,002,560 count= 37 avg_used= 2,054,123 block_cnt= 37 chunk_cnt= 37 mod=SqlPlanMon [MEMORY] hold= 62,373,888 used= 55,443,456 count= 846 avg_used= 65,536 block_cnt= 846 chunk_cnt= 165 mod=MajorMergeMgr [MEMORY] hold= 41,963,520 used= 41,943,040 count= 1 avg_used= 41,943,040 block_cnt= 1 chunk_cnt= 1 mod=LogGroupBuffer [MEMORY] hold= 39,350,272 used= 39,321,760 count= 2 avg_used= 19,660,880 block_cnt= 2 chunk_cnt= 2 mod=HashBuckDmId [MEMORY] hold= 37,769,216 used= 37,748,856 count= 1 avg_used= 37,748,856 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmChe [MEMORY] hold= 31,477,760 used= 31,458,304 count= 1 avg_used= 31,458,304 block_cnt= 1 chunk_cnt= 1 mod=ash_list [MEMORY] hold= 21,233,664 used= 18,959,616 count= 288 avg_used= 65,832 block_cnt= 288 chunk_cnt= 54 mod=[T]ObSessionDIB [MEMORY] hold= 18,874,368 used= 16,887,552 count= 256 avg_used= 65,967 block_cnt= 256 chunk_cnt= 69 mod=SqlDtlBuf [MEMORY] hold= 13,647,872 used= 13,603,118 count= 7 avg_used= 1,943,302 block_cnt= 7 chunk_cnt= 3 mod=LOCALDEVICE [MEMORY] hold= 13,074,080 used= 13,049,736 count= 60 avg_used= 217,495 block_cnt= 60 chunk_cnt= 17 mod=TabletMap [MEMORY] hold= 12,808,192 used= 12,763,552 count= 42 avg_used= 303,894 block_cnt= 42 chunk_cnt= 35 mod=IoControl [MEMORY] hold= 10,428,416 used= 10,387,456 count= 2 avg_used= 5,193,728 block_cnt= 2 chunk_cnt= 2 mod=ServerObjecPool [MEMORY] hold= 9,484,544 used= 9,332,736 count= 2,352 avg_used= 3,968 block_cnt= 1,182 chunk_cnt= 102 mod=CkpDgn [MEMORY] hold= 8,895,616 used= 8,810,249 count= 79 avg_used= 111,522 block_cnt= 27 chunk_cnt= 7 mod=OmtTenant [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=ArcFetchQueue [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=RCSrv [MEMORY] hold= 8,257,536 used= 8,240,064 count= 2 avg_used= 4,120,032 block_cnt= 2 chunk_cnt= 2 mod=FreeTbltStream [MEMORY] hold= 8,151,040 used= 8,131,072 count= 1 avg_used= 8,131,072 block_cnt= 1 chunk_cnt= 1 mod=TransService [MEMORY] hold= 7,881,232 used= 7,816,400 count= 1,007 avg_used= 7,762 block_cnt= 1,007 chunk_cnt= 134 mod=TLDecoderAlloc [MEMORY] hold= 7,827,648 used= 7,704,288 count= 1,706 avg_used= 4,515 block_cnt= 1,645 chunk_cnt= 161 mod=WorkerMap [MEMORY] hold= 7,815,168 used= 6,815,744 count= 122 avg_used= 55,866 block_cnt= 122 chunk_cnt= 44 mod=[T]char [MEMORY] hold= 5,554,624 used= 5,508,944 count= 707 avg_used= 7,792 block_cnt= 707 chunk_cnt= 97 mod=CkptDgnMemNode [MEMORY] hold= 4,739,072 used= 4,718,712 count= 1 avg_used= 4,718,712 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMonMap [MEMORY] hold= 4,161,536 used= 4,159,488 count= 2 avg_used= 2,079,744 block_cnt= 2 chunk_cnt= 2 mod=SqlExecutor [MEMORY] hold= 4,072,528 used= 2,305,320 count= 24,367 avg_used= 94 block_cnt= 63 chunk_cnt= 49 mod=TLDecoderCtx [MEMORY] hold= 4,066,688 used= 4,023,552 count= 476 avg_used= 8,452 block_cnt= 476 chunk_cnt= 134 mod=ParamCharsType [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMemCU [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 12 mod=CkptDgnMem [MEMORY] hold= 3,723,264 used= 3,698,688 count= 2 avg_used= 1,849,344 block_cnt= 2 chunk_cnt= 2 mod=LogDIOAligned [MEMORY] hold= 3,145,728 used= 2,228,224 count= 128 avg_used= 17,408 block_cnt= 128 chunk_cnt= 6 mod=SqlDtlQueue [MEMORY] hold= 2,981,888 used= 2,960,672 count= 13 avg_used= 227,744 block_cnt= 13 chunk_cnt= 12 mod=SQLSessionInfo [MEMORY] hold= 2,756,608 used= 2,720,005 count= 3 avg_used= 906,668 block_cnt= 3 chunk_cnt= 3 mod=SqlDtl1stBuf [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=TransTimeWheel [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=XATimeWheel [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=MediumTabletMap [MEMORY] hold= 2,375,680 used= 2,359,536 count= 2 avg_used= 1,179,768 block_cnt= 2 chunk_cnt= 2 mod=HashBuckLCSta [MEMORY] hold= 2,248,704 used= 2,228,224 count= 1 avg_used= 2,228,224 block_cnt= 1 chunk_cnt= 1 mod=LogIOCb [MEMORY] hold= 2,169,056 used= 408,600 count= 24,952 avg_used= 16 block_cnt= 266 chunk_cnt= 2 mod=Number [MEMORY] hold= 2,097,152 used= 2,093,056 count= 32 avg_used= 65,408 block_cnt= 32 chunk_cnt= 32 mod=StorMetaCacheIO [MEMORY] hold= 2,080,768 used= 2,079,744 count= 1 avg_used= 2,079,744 block_cnt= 1 chunk_cnt= 1 mod=LogReplayTask [MEMORY] hold= 1,966,080 used= 1,962,240 count= 30 avg_used= 65,408 block_cnt= 30 chunk_cnt= 7 mod=MemTblObj [MEMORY] hold= 1,777,664 used= 1,770,696 count= 1 avg_used= 1,770,696 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 1,670,592 used= 1,663,936 count= 7 avg_used= 237,705 block_cnt= 7 chunk_cnt= 2 mod=PoolFreeList [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=T3MBucket [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 1,433,600 used= 1,284,640 count= 35 avg_used= 36,704 block_cnt= 35 chunk_cnt= 31 mod=CommSysVarFac [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPlanCac [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPsInfo [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPsCache [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=RewriteRuleMap [MEMORY] hold= 1,048,576 used= 546,816 count= 64 avg_used= 8,544 block_cnt= 64 chunk_cnt= 26 mod=[T]ObDSActionAr [MEMORY] hold= 1,024,000 used= 1,015,808 count= 1 avg_used= 1,015,808 block_cnt= 1 chunk_cnt= 1 mod=CommitVersions [MEMORY] hold= 958,464 used= 950,272 count= 1 avg_used= 950,272 block_cnt= 1 chunk_cnt= 1 mod=IOWorkerLQ [MEMORY] hold= 933,888 used= 931,072 count= 1 avg_used= 931,072 block_cnt= 1 chunk_cnt= 1 mod=ArcSenderQueue [MEMORY] hold= 829,632 used= 804,864 count= 258 avg_used= 3,119 block_cnt= 130 chunk_cnt= 5 mod=LSMap [MEMORY] hold= 802,944 used= 786,432 count= 256 avg_used= 3,072 block_cnt= 128 chunk_cnt= 3 mod=TabletToLS [MEMORY] hold= 802,816 used= 800,000 count= 1 avg_used= 800,000 block_cnt= 1 chunk_cnt= 1 mod=SqlFltSpanRec [MEMORY] hold= 786,560 used= 524,352 count= 33 avg_used= 15,889 block_cnt= 33 chunk_cnt= 4 mod=LogAlloc [MEMORY] hold= 730,608 used= 724,656 count= 93 avg_used= 7,792 block_cnt= 93 chunk_cnt= 58 mod=CkptDgnCUNode [MEMORY] hold= 720,896 used= 712,704 count= 1 avg_used= 712,704 block_cnt= 1 chunk_cnt= 1 mod=DiskIteratorSto [MEMORY] hold= 673,296 used= 667,984 count= 83 avg_used= 8,048 block_cnt= 83 chunk_cnt= 46 mod=[T]ObPerfEventR [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 663,552 used= 655,360 count= 1 avg_used= 655,360 block_cnt= 1 chunk_cnt= 1 mod=FetchLog [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTR [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DupTbLease [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MultiVersionGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTF [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=OBJLockGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MdsT [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=ElectTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DetectorTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=FrzTrigger [MEMORY] hold= 608,000 used= 603,136 count= 76 avg_used= 7,936 block_cnt= 76 chunk_cnt= 45 mod=NotParamInfo [MEMORY] hold= 598,016 used= 590,232 count= 1 avg_used= 590,232 block_cnt= 1 chunk_cnt= 1 mod=DagNetIdMap [MEMORY] hold= 576,000 used= 571,392 count= 72 avg_used= 7,936 block_cnt= 72 chunk_cnt= 43 mod=SqlSession [MEMORY] hold= 524,288 used= 523,264 count= 8 avg_used= 65,408 block_cnt= 8 chunk_cnt= 5 mod=MemTblMgrObj [MEMORY] hold= 491,520 used= 474,240 count= 5 avg_used= 94,848 block_cnt= 5 chunk_cnt= 5 mod=ResourceGroup [MEMORY] hold= 451,584 used= 445,312 count= 98 avg_used= 4,544 block_cnt= 98 chunk_cnt= 46 mod=[T]ObTraceEvent [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ReplaySrv [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ApplySrv [MEMORY] hold= 385,024 used= 377,120 count= 1 avg_used= 377,120 block_cnt= 1 chunk_cnt= 1 mod=ClogGe [MEMORY] hold= 385,024 used= 375,520 count= 2 avg_used= 187,760 block_cnt= 2 chunk_cnt= 2 mod=bf_queue [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=DmlStatHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=ColUsagHashMap [MEMORY] hold= 280,128 used= 277,760 count= 35 avg_used= 7,936 block_cnt= 35 chunk_cnt= 24 mod=SqlSessiVarMap [MEMORY] hold= 280,000 used= 277,760 count= 35 avg_used= 7,936 block_cnt= 35 chunk_cnt= 32 mod=PlanVaIdx [MEMORY] hold= 256,128 used= 253,952 count= 32 avg_used= 7,936 block_cnt= 32 chunk_cnt= 30 mod=LSIter [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagNetMap [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagMap [MEMORY] hold= 200,704 used= 196,608 count= 64 avg_used= 3,072 block_cnt= 32 chunk_cnt= 3 mod=TxCtxMgr [MEMORY] hold= 169,344 used= 166,656 count= 42 avg_used= 3,968 block_cnt= 22 chunk_cnt= 14 mod=TinyAllocator [MEMORY] hold= 147,456 used= 139,264 count= 1 avg_used= 139,264 block_cnt= 1 chunk_cnt= 1 mod=RFLTaskQueue [MEMORY] hold= 141,696 used= 140,544 count= 18 avg_used= 7,808 block_cnt= 18 chunk_cnt= 15 mod=HashNodeLCSta [MEMORY] hold= 122,880 used= 110,608 count= 2 avg_used= 55,304 block_cnt= 2 chunk_cnt= 1 mod=TabletIDSetBkt [MEMORY] hold= 114,688 used= 106,496 count= 1 avg_used= 106,496 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlMgr [MEMORY] hold= 98,304 used= 94,976 count= 1 avg_used= 94,976 block_cnt= 1 chunk_cnt= 1 mod=PxPool [MEMORY] hold= 90,112 used= 84,448 count= 1 avg_used= 84,448 block_cnt= 1 chunk_cnt= 1 mod=LSSvr [MEMORY] hold= 90,112 used= 82,048 count= 1 avg_used= 82,048 block_cnt= 1 chunk_cnt= 1 mod=MetaMemMgr [MEMORY] hold= 74,160 used= 35,568 count= 525 avg_used= 67 block_cnt= 185 chunk_cnt= 38 mod=Coro [MEMORY] hold= 73,728 used= 72,736 count= 1 avg_used= 72,736 block_cnt= 1 chunk_cnt= 1 mod=LogSharedQueueT [MEMORY] hold= 73,728 used= 66,176 count= 1 avg_used= 66,176 block_cnt= 1 chunk_cnt= 1 mod=Rpc [MEMORY] hold= 73,728 used= 65,536 count= 1 avg_used= 65,536 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoMgr [MEMORY] hold= 72,240 used= 1,024 count= 896 avg_used= 1 block_cnt= 10 chunk_cnt= 1 mod=CharsetUtil [MEMORY] hold= 70,848 used= 70,272 count= 9 avg_used= 7,808 block_cnt= 9 chunk_cnt= 9 mod=HashNodePlanCac [MEMORY] hold= 70,848 used= 70,272 count= 9 avg_used= 7,808 block_cnt= 9 chunk_cnt= 8 mod=TabletIDSetNode [MEMORY] hold= 67,920 used= 61,824 count= 92 avg_used= 672 block_cnt= 67 chunk_cnt= 38 mod=[T]ObWarningBuf [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=FlushLog [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=DEVICE_MANAGER [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxDataMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxCtxMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LockMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=FlushMeta [MEMORY] hold= 59,040 used= 50,184 count= 123 avg_used= 408 block_cnt= 120 chunk_cnt= 57 mod=[T]ObTLDecoderC [MEMORY] hold= 49,152 used= 48,768 count= 1 avg_used= 48,768 block_cnt= 1 chunk_cnt= 1 mod=palf_env [MEMORY] hold= 48,832 used= 44,840 count= 2 avg_used= 22,420 block_cnt= 2 chunk_cnt= 2 mod=DynamicFactor [MEMORY] hold= 42,864 used= 37,024 count= 89 avg_used= 416 block_cnt= 37 chunk_cnt= 13 mod=OMT_Worker [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HB_SERVICE [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=Autoincrement [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisBkt [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 1 mod=ObDMMDL [MEMORY] hold= 35,200 used= 34,560 count= 10 avg_used= 3,456 block_cnt= 10 chunk_cnt= 6 mod=CACHE_INST [MEMORY] hold= 32,768 used= 24,688 count= 2 avg_used= 12,344 block_cnt= 2 chunk_cnt= 1 mod=TLD_ClientTask [MEMORY] hold= 32,768 used= 17,408 count= 2 avg_used= 8,704 block_cnt= 2 chunk_cnt= 2 mod=ObLogEXTTP [MEMORY] hold= 32,768 used= 25,600 count= 1 avg_used= 25,600 block_cnt= 1 chunk_cnt= 1 mod=TSQLSessionMgr [MEMORY] hold= 32,000 used= 31,744 count= 4 avg_used= 7,936 block_cnt= 4 chunk_cnt= 4 mod=MdsTableMgr [MEMORY] hold= 28,400 used= 6,912 count= 288 avg_used= 24 block_cnt= 116 chunk_cnt= 47 mod=[T]MemoryContex [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 24,240 used= 20,136 count= 2 avg_used= 10,068 block_cnt= 2 chunk_cnt= 2 mod=TabletStats [MEMORY] hold= 23,280 used= 20,980 count= 35 avg_used= 599 block_cnt= 33 chunk_cnt= 24 mod=SqlSessiQuerSql [MEMORY] hold= 22,848 used= 21,504 count= 21 avg_used= 1,024 block_cnt= 20 chunk_cnt= 5 mod=TGTimer [MEMORY] hold= 19,648 used= 16,832 count= 44 avg_used= 382 block_cnt= 41 chunk_cnt= 37 mod=NegParamIdex [MEMORY] hold= 17,504 used= 14,840 count= 35 avg_used= 424 block_cnt= 34 chunk_cnt= 24 mod=SequenceMap [MEMORY] hold= 17,392 used= 14,840 count= 35 avg_used= 424 block_cnt= 33 chunk_cnt= 25 mod=PackStateMap [MEMORY] hold= 17,376 used= 14,840 count= 35 avg_used= 424 block_cnt= 33 chunk_cnt= 28 mod=ContextsMap [MEMORY] hold= 17,376 used= 14,840 count= 35 avg_used= 424 block_cnt= 35 chunk_cnt= 26 mod=SequenceIdMap [MEMORY] hold= 16,768 used= 8,448 count= 3 avg_used= 2,816 block_cnt= 2 chunk_cnt= 2 mod=BaseLogWriter [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TableCtxMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=backupTaskSched [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TblCtxIMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_AssignedMgr [MEMORY] hold= 16,384 used= 8,448 count= 1 avg_used= 8,448 block_cnt= 1 chunk_cnt= 1 mod=PalfEnv [MEMORY] hold= 16,384 used= 11,584 count= 1 avg_used= 11,584 block_cnt= 1 chunk_cnt= 1 mod=LogApplyStatus [MEMORY] hold= 16,384 used= 13,696 count= 1 avg_used= 13,696 block_cnt= 1 chunk_cnt= 1 mod=LogReplayStatus [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 10,248 count= 1 avg_used= 10,248 block_cnt= 1 chunk_cnt= 1 mod=TabletSvrBucket [MEMORY] hold= 15,936 used= 15,808 count= 2 avg_used= 7,904 block_cnt= 2 chunk_cnt= 2 mod=SqlSessionSbloc [MEMORY] hold= 12,576 used= 12,440 count= 2 avg_used= 6,220 block_cnt= 2 chunk_cnt= 2 mod=PxPoolBkt [MEMORY] hold= 10,240 used= 10,112 count= 2 avg_used= 5,056 block_cnt= 1 chunk_cnt= 1 mod=Scheduler [MEMORY] hold= 9,536 used= 6,016 count= 55 avg_used= 109 block_cnt= 55 chunk_cnt= 34 mod=NotParamIdex [MEMORY] hold= 8,192 used= 7,936 count= 4 avg_used= 1,984 block_cnt= 2 chunk_cnt= 2 mod=CreateContext [MEMORY] hold= 8,192 used= 8,056 count= 2 avg_used= 4,028 block_cnt= 2 chunk_cnt= 2 mod=StoLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=APPLY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LockWaitMgr [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LSLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ShareBlksMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MdsDebugMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=REPLAY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MatchLsnMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ls_id [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LCLSender [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LsElectRefInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DASIDCache [MEMORY] hold= 7,968 used= 7,904 count= 1 avg_used= 7,904 block_cnt= 1 chunk_cnt= 1 mod=PsSessionInfo [MEMORY] hold= 7,872 used= 7,808 count= 1 avg_used= 7,808 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoNode [MEMORY] hold= 7,872 used= 7,808 count= 1 avg_used= 7,808 block_cnt= 1 chunk_cnt= 1 mod=HashNodePsInfo [MEMORY] hold= 7,856 used= 7,792 count= 1 avg_used= 7,792 block_cnt= 1 chunk_cnt= 1 mod=ColUsagNode [MEMORY] hold= 7,792 used= 7,728 count= 1 avg_used= 7,728 block_cnt= 1 chunk_cnt= 1 mod=HashNodePsCache [MEMORY] hold= 6,416 used= 6,216 count= 3 avg_used= 2,072 block_cnt= 2 chunk_cnt= 2 mod=DetectManager [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResGrpIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResRuleIdMap [MEMORY] hold= 4,160 used= 4,096 count= 1 avg_used= 4,096 block_cnt= 1 chunk_cnt= 1 mod=MacroFile [MEMORY] hold= 3,776 used= 3,712 count= 1 avg_used= 3,712 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlDfc [MEMORY] hold= 3,136 used= 3,072 count= 1 avg_used= 3,072 block_cnt= 1 chunk_cnt= 1 mod=LockMemtableMgr [MEMORY] hold= 2,576 used= 64 count= 32 avg_used= 2 block_cnt= 31 chunk_cnt= 22 mod=[T]TSIUseWeak [MEMORY] hold= 2,336 used= 1,792 count= 8 avg_used= 224 block_cnt= 3 chunk_cnt= 3 mod=LogIOTask [MEMORY] hold= 2,240 used= 1,920 count= 5 avg_used= 384 block_cnt= 5 chunk_cnt= 5 mod=TableLockNode [MEMORY] hold= 2,224 used= 1,664 count= 8 avg_used= 208 block_cnt= 8 chunk_cnt= 7 mod=[T]ObMacroBlock [MEMORY] hold= 1,920 used= 1,800 count= 1 avg_used= 1,800 block_cnt= 1 chunk_cnt= 1 mod=PxResMgr [MEMORY] hold= 1,920 used= 1,856 count= 1 avg_used= 1,856 block_cnt= 1 chunk_cnt= 1 mod=LogIOWS [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=MSTXCTX [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TabStatMgrLock [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TXBFRetryId [MEMORY] hold= 1,408 used= 1,280 count= 2 avg_used= 640 block_cnt= 1 chunk_cnt= 1 mod=PoolArenaArray [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GCMemtableMap [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=DUP_LS_SET [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 1,120 used= 1,056 count= 1 avg_used= 1,056 block_cnt= 1 chunk_cnt= 1 mod=LOG_HASH_MAP [MEMORY] hold= 1,120 used= 720 count= 5 avg_used= 144 block_cnt= 5 chunk_cnt= 1 mod=ObTxLSLogCb [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=DiskUsageTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=FlushTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TabShellTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=CheckPointTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TLD_TIMER [MEMORY] hold= 848 used= 776 count= 1 avg_used= 776 block_cnt= 1 chunk_cnt= 1 mod=HashBucPsSessIn [MEMORY] hold= 848 used= 776 count= 1 avg_used= 776 block_cnt= 1 chunk_cnt= 1 mod=DepTabletSet [MEMORY] hold= 720 used= 648 count= 1 avg_used= 648 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfo [MEMORY] hold= 624 used= 552 count= 1 avg_used= 552 block_cnt= 1 chunk_cnt= 1 mod=LSFreeze [MEMORY] hold= 576 used= 472 count= 1 avg_used= 472 block_cnt= 1 chunk_cnt= 1 mod=Election [MEMORY] hold= 528 used= 120 count= 5 avg_used= 24 block_cnt= 2 chunk_cnt= 1 mod=FreezeTask [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=HAScheduler [MEMORY] hold= 336 used= 144 count= 2 avg_used= 72 block_cnt= 2 chunk_cnt= 2 mod=TntSrvObjPool [MEMORY] hold= 192 used= 104 count= 1 avg_used= 104 block_cnt= 1 chunk_cnt= 1 mod=RedoLogBuckLock [MEMORY] hold= 192 used= 64 count= 2 avg_used= 32 block_cnt= 1 chunk_cnt= 1 mod=PalfFSCbNode [MEMORY] hold= 176 used= 112 count= 1 avg_used= 112 block_cnt= 1 chunk_cnt= 1 mod=Coordinator [MEMORY] hold= 160 used= 16 count= 2 avg_used= 8 block_cnt= 2 chunk_cnt= 2 mod=ObLogEXTHandler [MEMORY] hold= 128 used= 56 count= 1 avg_used= 56 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfoMap [MEMORY] hold= 112 used= 41 count= 1 avg_used= 41 block_cnt= 1 chunk_cnt= 1 mod=KeepAliveBuf [MEMORY] hold= 96 used= 32 count= 1 avg_used= 32 block_cnt= 1 chunk_cnt= 1 mod=RebuildCbNode [MEMORY] hold= 96 used= 32 count= 1 avg_used= 32 block_cnt= 1 chunk_cnt= 1 mod=PalfRCCbNode [MEMORY] hold= 80 used= 16 count= 1 avg_used= 16 block_cnt= 1 chunk_cnt= 1 mod=logservice [MEMORY] hold= 1,617,364,480 used= 1,582,025,881 count= 65,216 avg_used= 24,258 mod=SUMMARY [2024-06-25 17:00:18.759576] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=28] [MEMORY] tenant_id= 1 ctx_id= MEMSTORE_CTX_ID hold= 285,212,672 used= 282,984,448 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 282,984,448 used= 282,854,976 count= 136 avg_used= 2,079,816 block_cnt= 136 chunk_cnt= 136 mod=Memstore [MEMORY] hold= 282,984,448 used= 282,854,976 count= 136 avg_used= 2,079,816 mod=SUMMARY [2024-06-25 17:00:18.759584] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1 ctx_id= TRANS_CTX_MGR_ID hold= 2,097,152 used= 729,088 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 729,088 used= 722,496 count= 1 avg_used= 722,496 block_cnt= 1 chunk_cnt= 1 mod=PartTranCtxMgr [MEMORY] hold= 729,088 used= 722,496 count= 1 avg_used= 722,496 mod=SUMMARY [2024-06-25 17:00:18.759589] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.759593] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1 ctx_id= PLAN_CACHE_CTX_ID hold= 312,475,648 used= 291,707,968 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 269,003,072 used= 247,743,202 count= 10,744 avg_used= 23,058 block_cnt= 10,744 chunk_cnt= 149 mod=SqlPhyPlan [MEMORY] hold= 16,283,584 used= 15,003,888 count= 924 avg_used= 16,237 block_cnt= 924 chunk_cnt= 147 mod=SqlPlanCache [MEMORY] hold= 3,742,464 used= 3,624,768 count= 1,827 avg_used= 1,984 block_cnt= 459 chunk_cnt= 132 mod=CreateContext [MEMORY] hold= 1,440,000 used= 1,428,480 count= 180 avg_used= 7,936 block_cnt= 180 chunk_cnt= 40 mod=OB_SQLSTAT_OBJ [MEMORY] hold= 1,237,120 used= 1,196,088 count= 140 avg_used= 8,543 block_cnt= 140 chunk_cnt= 2 mod=SqlPhyPlObj [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=PlanCache [MEMORY] hold= 291,707,968 used= 268,998,026 count= 13,817 avg_used= 19,468 mod=SUMMARY [2024-06-25 17:00:18.759598] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.759603] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1 ctx_id= GLIBC hold= 4,194,304 used= 2,409,008 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 2,360,128 used= 1,878,143 count= 6,788 avg_used= 276 block_cnt= 296 chunk_cnt= 2 mod=PlJit [MEMORY] hold= 48,880 used= 45,842 count= 43 avg_used= 1,066 block_cnt= 13 chunk_cnt= 1 mod=PlCodeGen [MEMORY] hold= 2,409,008 used= 1,923,985 count= 6,831 avg_used= 281 mod=SUMMARY [2024-06-25 17:00:18.759603] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.759608] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.759611] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1 ctx_id= CO_STACK hold= 146,800,640 used= 143,990,784 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 143,990,784 used= 143,722,944 count= 279 avg_used= 515,136 block_cnt= 279 chunk_cnt= 70 mod=CoStack [MEMORY] hold= 143,990,784 used= 143,722,944 count= 279 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.759612] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.759621] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1 ctx_id= KVSTORE_CACHE_ID hold= 14,854,127,616 used= 14,854,127,616 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 14,854,127,616 used= 0 count= 7,083 avg_used= 0 block_cnt= 0 chunk_cnt= 0 mod=KvstorCacheMb [MEMORY] hold= 14,854,127,616 used= 0 count= 7,083 avg_used= 0 mod=SUMMARY [2024-06-25 17:00:18.759630] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1 ctx_id= META_OBJ_CTX_ID hold= 20,971,520 used= 18,849,792 limit= 8,079,907,220 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 13,271,040 used= 11,806,380 count= 180 avg_used= 65,591 block_cnt= 180 chunk_cnt= 8 mod=L_TabletPool [MEMORY] hold= 5,177,344 used= 5,167,232 count= 79 avg_used= 65,408 block_cnt= 79 chunk_cnt= 5 mod=N_TabletPool [MEMORY] hold= 401,408 used= 400,064 count= 2 avg_used= 200,032 block_cnt= 2 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 18,849,792 used= 17,373,676 count= 261 avg_used= 66,565 mod=SUMMARY [2024-06-25 17:00:18.759638] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1 ctx_id= TX_CALLBACK_CTX_ID hold= 20,971,520 used= 16,232,768 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 16,232,768 used= 16,102,144 count= 2,029 avg_used= 7,936 block_cnt= 2,029 chunk_cnt= 10 mod=MemtableCallbac [MEMORY] hold= 16,232,768 used= 16,102,144 count= 2,029 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.759646] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1 ctx_id= LOB_CTX_ID hold= 2,097,152 used= 65,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LobAllocator [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 mod=SUMMARY [2024-06-25 17:00:18.759652] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1 ctx_id= PS_CACHE_CTX_ID hold= 2,097,152 used= 26,624 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 26,624 used= 25,214 count= 20 avg_used= 1,260 block_cnt= 5 chunk_cnt= 1 mod=SqlPsCache [MEMORY] hold= 26,624 used= 25,214 count= 20 avg_used= 1,260 mod=SUMMARY [2024-06-25 17:00:18.759664] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1 ctx_id= TX_DATA_TABLE hold= 69,226,496 used= 64,977,280 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 47,106,816 used= 46,727,168 count= 5,888 avg_used= 7,936 block_cnt= 5,888 chunk_cnt= 25 mod=TX_DATA_SLICE [MEMORY] hold= 17,862,464 used= 17,833,792 count= 3 avg_used= 5,944,597 block_cnt= 3 chunk_cnt= 3 mod=MEMTABLE_ARENA [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=TX_DATA_ARENA [MEMORY] hold= 64,977,280 used= 64,568,896 count= 5,892 avg_used= 10,958 mod=SUMMARY [2024-06-25 17:00:18.759674] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1 ctx_id= MDS_DATA_ID hold= 2,097,152 used= 64,000 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 64,000 used= 63,488 count= 8 avg_used= 7,936 block_cnt= 8 chunk_cnt= 1 mod=MdsTable [MEMORY] hold= 64,000 used= 63,488 count= 8 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.759695] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant: 1001, limit: 1,073,741,824 hold: 343,166,976 rpc_hold: 0 cache_hold: 14,680,064 cache_used: 14,680,064 cache_item_count: 7 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 183,783,424 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PLAN_CACHE_CTX_ID hold_bytes= 48,234,496 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= GLIBC hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 88,080,384 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= META_OBJ_CTX_ID hold_bytes= 2,097,152 limit= 214,748,360 [MEMORY] ctx_id= LOB_CTX_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY][PM] tid= 63495 used= 0 hold= 2,097,152 pm=0x2b3aa75d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 63496 used= 0 hold= 4,194,304 pm=0x2b3aa7656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 63497 used= 0 hold= 4,194,304 pm=0x2b3aa76d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 63499 used= 0 hold= 2,097,152 pm=0x2b3aa77d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= summary used= 0 hold= 12,582,912 [2024-06-25 17:00:18.759796] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1001 ctx_id= DEFAULT_CTX_ID hold= 183,783,424 used= 139,026,848 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 13,303,808 used= 13,289,408 count= 8 avg_used= 1,661,176 block_cnt= 8 chunk_cnt= 8 mod=MysqlRequesReco [MEMORY] hold= 12,091,392 used= 10,796,448 count= 164 avg_used= 65,832 block_cnt= 164 chunk_cnt= 21 mod=[T]ObSessionDIB [MEMORY] hold= 10,711,040 used= 10,670,496 count= 10 avg_used= 1,067,049 block_cnt= 10 chunk_cnt= 6 mod=IoControl [MEMORY] hold= 8,895,632 used= 8,810,249 count= 79 avg_used= 111,522 block_cnt= 25 chunk_cnt= 5 mod=OmtTenant [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=ArcFetchQueue [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=RCSrv [MEMORY] hold= 8,257,536 used= 8,240,064 count= 2 avg_used= 4,120,032 block_cnt= 2 chunk_cnt= 2 mod=FreeTbltStream [MEMORY] hold= 6,561,792 used= 5,871,063 count= 89 avg_used= 65,967 block_cnt= 89 chunk_cnt= 6 mod=SqlDtlBuf [MEMORY] hold= 4,161,536 used= 4,159,488 count= 2 avg_used= 2,079,744 block_cnt= 2 chunk_cnt= 2 mod=CACHE_MAP_NODE [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMem [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 10 mod=CkptDgnMemCU [MEMORY] hold= 3,145,728 used= 2,228,224 count= 128 avg_used= 17,408 block_cnt= 128 chunk_cnt= 5 mod=SqlDtlQueue [MEMORY] hold= 2,756,608 used= 2,720,005 count= 3 avg_used= 906,668 block_cnt= 3 chunk_cnt= 2 mod=SqlDtl1stBuf [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=TransTimeWheel [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=XATimeWheel [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=MediumTabletMap [MEMORY] hold= 2,375,680 used= 2,359,536 count= 2 avg_used= 1,179,768 block_cnt= 2 chunk_cnt= 2 mod=HashBuckLCSta [MEMORY] hold= 2,248,704 used= 2,228,224 count= 1 avg_used= 2,228,224 block_cnt= 1 chunk_cnt= 1 mod=LogIOCb [MEMORY] hold= 1,777,664 used= 1,770,696 count= 1 avg_used= 1,770,696 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 1,744,896 used= 1,540,096 count= 25 avg_used= 61,603 block_cnt= 25 chunk_cnt= 8 mod=[T]char [MEMORY] hold= 1,213,760 used= 1,195,056 count= 258 avg_used= 4,632 block_cnt= 258 chunk_cnt= 23 mod=WorkerMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=RewriteRuleMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMonMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPlanCac [MEMORY] hold= 958,464 used= 950,272 count= 1 avg_used= 950,272 block_cnt= 1 chunk_cnt= 1 mod=IOWorkerLQ [MEMORY] hold= 933,888 used= 931,072 count= 1 avg_used= 931,072 block_cnt= 1 chunk_cnt= 1 mod=ArcSenderQueue [MEMORY] hold= 829,632 used= 804,864 count= 258 avg_used= 3,119 block_cnt= 130 chunk_cnt= 2 mod=LSMap [MEMORY] hold= 802,944 used= 786,432 count= 256 avg_used= 3,072 block_cnt= 128 chunk_cnt= 4 mod=TabletToLS [MEMORY] hold= 802,816 used= 800,000 count= 1 avg_used= 800,000 block_cnt= 1 chunk_cnt= 1 mod=SqlFltSpanRec [MEMORY] hold= 794,624 used= 786,536 count= 1 avg_used= 786,536 block_cnt= 1 chunk_cnt= 1 mod=TabletMap [MEMORY] hold= 786,560 used= 524,352 count= 33 avg_used= 15,889 block_cnt= 33 chunk_cnt= 1 mod=LogAlloc [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 663,552 used= 655,360 count= 1 avg_used= 655,360 block_cnt= 1 chunk_cnt= 1 mod=FetchLog [MEMORY] hold= 655,360 used= 649,216 count= 2 avg_used= 324,608 block_cnt= 2 chunk_cnt= 1 mod=ServerObjecPool [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTF [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=ElectTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=FrzTrigger [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MdsT [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DetectorTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTR [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MultiVersionGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DupTbLease [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=OBJLockGC [MEMORY] hold= 598,016 used= 590,232 count= 1 avg_used= 590,232 block_cnt= 1 chunk_cnt= 1 mod=DagNetIdMap [MEMORY] hold= 532,480 used= 531,072 count= 1 avg_used= 531,072 block_cnt= 1 chunk_cnt= 1 mod=TransService [MEMORY] hold= 523,776 used= 523,264 count= 8 avg_used= 65,408 block_cnt= 8 chunk_cnt= 6 mod=ResultSet [MEMORY] hold= 488,000 used= 484,096 count= 61 avg_used= 7,936 block_cnt= 61 chunk_cnt= 16 mod=ParamCharsType [MEMORY] hold= 434,176 used= 416,870 count= 4 avg_used= 104,217 block_cnt= 4 chunk_cnt= 2 mod=SqlMemMgr [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ApplySrv [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ReplaySrv [MEMORY] hold= 392,640 used= 383,936 count= 7 avg_used= 54,848 block_cnt= 7 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 385,024 used= 375,520 count= 2 avg_used= 187,760 block_cnt= 2 chunk_cnt= 1 mod=bf_queue [MEMORY] hold= 327,680 used= 325,376 count= 1 avg_used= 325,376 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMon [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=DmlStatHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=ColUsagHashMap [MEMORY] hold= 229,376 used= 227,744 count= 1 avg_used= 227,744 block_cnt= 1 chunk_cnt= 1 mod=SQLSessionInfo [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagMap [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagNetMap [MEMORY] hold= 200,704 used= 196,608 count= 64 avg_used= 3,072 block_cnt= 32 chunk_cnt= 4 mod=TxCtxMgr [MEMORY] hold= 147,456 used= 139,264 count= 1 avg_used= 139,264 block_cnt= 1 chunk_cnt= 1 mod=RFLTaskQueue [MEMORY] hold= 114,688 used= 106,496 count= 1 avg_used= 106,496 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlMgr [MEMORY] hold= 106,496 used= 98,320 count= 1 avg_used= 98,320 block_cnt= 1 chunk_cnt= 1 mod=T3MBucket [MEMORY] hold= 106,496 used= 98,320 count= 1 avg_used= 98,320 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 101,152 used= 100,320 count= 13 avg_used= 7,716 block_cnt= 13 chunk_cnt= 4 mod=TLDecoderAlloc [MEMORY] hold= 98,304 used= 94,848 count= 1 avg_used= 94,848 block_cnt= 1 chunk_cnt= 1 mod=ResourceGroup [MEMORY] hold= 90,112 used= 82,048 count= 1 avg_used= 82,048 block_cnt= 1 chunk_cnt= 1 mod=MetaMemMgr [MEMORY] hold= 81,920 used= 42,720 count= 5 avg_used= 8,544 block_cnt= 5 chunk_cnt= 5 mod=[T]ObDSActionAr [MEMORY] hold= 73,728 used= 72,736 count= 1 avg_used= 72,736 block_cnt= 1 chunk_cnt= 1 mod=LogSharedQueueT [MEMORY] hold= 56,784 used= 56,336 count= 7 avg_used= 8,048 block_cnt= 7 chunk_cnt= 6 mod=[T]ObPerfEventR [MEMORY] hold= 48,832 used= 44,840 count= 2 avg_used= 22,420 block_cnt= 2 chunk_cnt= 2 mod=DynamicFactor [MEMORY] hold= 45,200 used= 21,136 count= 327 avg_used= 64 block_cnt= 54 chunk_cnt= 22 mod=Coro [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=Autoincrement [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisBkt [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 5 mod=NotParamInfo [MEMORY] hold= 32,768 used= 25,600 count= 1 avg_used= 25,600 block_cnt= 1 chunk_cnt= 1 mod=TSQLSessionMgr [MEMORY] hold= 32,768 used= 17,408 count= 2 avg_used= 8,704 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTTP [MEMORY] hold= 32,768 used= 24,688 count= 2 avg_used= 12,344 block_cnt= 2 chunk_cnt= 1 mod=TLD_ClientTask [MEMORY] hold= 32,256 used= 31,808 count= 7 avg_used= 4,544 block_cnt= 7 chunk_cnt= 5 mod=[T]ObTraceEvent [MEMORY] hold= 31,488 used= 31,232 count= 4 avg_used= 7,808 block_cnt= 4 chunk_cnt= 3 mod=HashNodeLCSta [MEMORY] hold= 25,424 used= 19,232 count= 2 avg_used= 9,616 block_cnt= 2 chunk_cnt= 1 mod=HashBuckDmId [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmChe [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 21,760 used= 20,480 count= 20 avg_used= 1,024 block_cnt= 17 chunk_cnt= 11 mod=TGTimer [MEMORY] hold= 16,768 used= 8,448 count= 3 avg_used= 2,816 block_cnt= 2 chunk_cnt= 1 mod=BaseLogWriter [MEMORY] hold= 16,384 used= 8,448 count= 1 avg_used= 8,448 block_cnt= 1 chunk_cnt= 1 mod=PalfEnv [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TableCtxMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=backupTaskSched [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TabletStats [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_AssignedMgr [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TblCtxIMgr [MEMORY] hold= 15,936 used= 3,936 count= 164 avg_used= 24 block_cnt= 44 chunk_cnt= 20 mod=[T]MemoryContex [MEMORY] hold= 15,744 used= 15,616 count= 2 avg_used= 7,808 block_cnt= 2 chunk_cnt= 2 mod=HashNodePlanCac [MEMORY] hold= 12,912 used= 8,664 count= 59 avg_used= 146 block_cnt= 1 chunk_cnt= 1 mod=TLDecoderCtx [MEMORY] hold= 8,192 used= 8,056 count= 2 avg_used= 4,028 block_cnt= 2 chunk_cnt= 1 mod=StoLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LCLSender [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ShareBlksMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LSLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LSIter [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LsElectRefInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MdsDebugMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=REPLAY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=APPLY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DASIDCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ObDMMDL [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LockWaitMgr [MEMORY] hold= 7,088 used= 6,912 count= 2 avg_used= 3,456 block_cnt= 2 chunk_cnt= 2 mod=CACHE_INST [MEMORY] hold= 6,416 used= 6,216 count= 3 avg_used= 2,072 block_cnt= 2 chunk_cnt= 2 mod=DetectManager [MEMORY] hold= 5,152 used= 4,704 count= 7 avg_used= 672 block_cnt= 4 chunk_cnt= 4 mod=[T]ObWarningBuf [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResRuleIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=PxPoolBkt [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResGrpIdMap [MEMORY] hold= 4,160 used= 4,096 count= 1 avg_used= 4,096 block_cnt= 1 chunk_cnt= 1 mod=MacroFile [MEMORY] hold= 3,776 used= 3,712 count= 1 avg_used= 3,712 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlDfc [MEMORY] hold= 3,360 used= 2,912 count= 7 avg_used= 416 block_cnt= 5 chunk_cnt= 4 mod=OMT_Worker [MEMORY] hold= 2,304 used= 1,792 count= 8 avg_used= 224 block_cnt= 1 chunk_cnt= 1 mod=LogIOTask [MEMORY] hold= 1,920 used= 1,800 count= 1 avg_used= 1,800 block_cnt= 1 chunk_cnt= 1 mod=PxResMgr [MEMORY] hold= 1,920 used= 1,856 count= 1 avg_used= 1,856 block_cnt= 1 chunk_cnt= 1 mod=LogIOWS [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TabStatMgrLock [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GCMemtableMap [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=DUP_LS_SET [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 1,120 used= 1,056 count= 1 avg_used= 1,056 block_cnt= 1 chunk_cnt= 1 mod=LOG_HASH_MAP [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TLD_TIMER [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TabShellTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=CheckPointTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=DiskUsageTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=FlushTimer [MEMORY] hold= 960 used= 816 count= 2 avg_used= 408 block_cnt= 2 chunk_cnt= 2 mod=[T]ObTLDecoderC [MEMORY] hold= 768 used= 640 count= 2 avg_used= 320 block_cnt= 1 chunk_cnt= 1 mod=PoolArenaArray [MEMORY] hold= 720 used= 648 count= 1 avg_used= 648 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfo [MEMORY] hold= 624 used= 552 count= 1 avg_used= 552 block_cnt= 1 chunk_cnt= 1 mod=LSFreeze [MEMORY] hold= 480 used= 120 count= 5 avg_used= 24 block_cnt= 1 chunk_cnt= 1 mod=FreezeTask [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=Scheduler [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=HAScheduler [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=MSTXCTX [MEMORY] hold= 336 used= 144 count= 2 avg_used= 72 block_cnt= 2 chunk_cnt= 1 mod=TntSrvObjPool [MEMORY] hold= 240 used= 6 count= 3 avg_used= 2 block_cnt= 3 chunk_cnt= 3 mod=[T]TSIUseWeak [MEMORY] hold= 192 used= 128 count= 1 avg_used= 128 block_cnt= 1 chunk_cnt= 1 mod=NegParamIdex [MEMORY] hold= 176 used= 112 count= 1 avg_used= 112 block_cnt= 1 chunk_cnt= 1 mod=Coordinator [MEMORY] hold= 160 used= 16 count= 2 avg_used= 8 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTHandler [MEMORY] hold= 128 used= 56 count= 1 avg_used= 56 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfoMap [MEMORY] hold= 80 used= 16 count= 1 avg_used= 16 block_cnt= 1 chunk_cnt= 1 mod=logservice [MEMORY] hold= 139,026,848 used= 134,626,521 count= 3,835 avg_used= 35,104 mod=SUMMARY [2024-06-25 17:00:18.759829] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=20] [MEMORY] tenant_id= 1001 ctx_id= PLAN_CACHE_CTX_ID hold= 48,234,496 used= 36,767,664 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 33,025,072 used= 30,293,379 count= 1,440 avg_used= 21,037 block_cnt= 1,440 chunk_cnt= 22 mod=SqlPhyPlan [MEMORY] hold= 1,384,000 used= 1,307,704 count= 146 avg_used= 8,956 block_cnt= 146 chunk_cnt= 20 mod=SqlPlanCache [MEMORY] hold= 1,237,120 used= 1,196,088 count= 140 avg_used= 8,543 block_cnt= 140 chunk_cnt= 3 mod=SqlPhyPlObj [MEMORY] hold= 655,488 used= 634,880 count= 320 avg_used= 1,984 block_cnt= 81 chunk_cnt= 20 mod=CreateContext [MEMORY] hold= 464,256 used= 460,288 count= 58 avg_used= 7,936 block_cnt= 58 chunk_cnt= 13 mod=OB_SQLSTAT_OBJ [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=PlanCache [MEMORY] hold= 36,767,664 used= 33,893,939 count= 2,106 avg_used= 16,093 mod=SUMMARY [2024-06-25 17:00:18.759840] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1001 ctx_id= GLIBC hold= 4,194,304 used= 2,327,680 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 2,278,848 used= 1,819,125 count= 6,525 avg_used= 278 block_cnt= 288 chunk_cnt= 2 mod=PlJit [MEMORY] hold= 48,832 used= 45,810 count= 42 avg_used= 1,090 block_cnt= 13 chunk_cnt= 1 mod=PlCodeGen [MEMORY] hold= 2,327,680 used= 1,864,935 count= 6,567 avg_used= 283 mod=SUMMARY [2024-06-25 17:00:18.759848] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1001 ctx_id= CO_STACK hold= 88,080,384 used= 85,155,840 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 85,155,840 used= 84,997,440 count= 165 avg_used= 515,136 block_cnt= 165 chunk_cnt= 42 mod=CoStack [MEMORY] hold= 85,155,840 used= 84,997,440 count= 165 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.759858] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1001 ctx_id= KVSTORE_CACHE_ID hold= 14,680,064 used= 14,680,064 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 14,680,064 used= 0 count= 7 avg_used= 0 block_cnt= 0 chunk_cnt= 0 mod=KvstorCacheMb [MEMORY] hold= 14,680,064 used= 0 count= 7 avg_used= 0 mod=SUMMARY [2024-06-25 17:00:18.759855] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164325C300C3-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301544, send_timestamp=1719305996458310) [2024-06-25 17:00:18.759866] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1001 ctx_id= META_OBJ_CTX_ID hold= 2,097,152 used= 89,920 limit= 214,748,360 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 89,920 used= 84,768 count= 2 avg_used= 42,384 block_cnt= 2 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 89,920 used= 84,768 count= 2 avg_used= 42,384 mod=SUMMARY [2024-06-25 17:00:18.759876] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1001 ctx_id= LOB_CTX_ID hold= 2,097,152 used= 65,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LobAllocator [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 mod=SUMMARY [2024-06-25 17:00:18.759915] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant: 1002, limit: 1,073,741,824 hold: 933,953,536 rpc_hold: 0 cache_hold: 123,731,968 cache_used: 123,731,968 cache_item_count: 59 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 548,995,072 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MEMSTORE_CTX_ID hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= EXECUTE_CTX_ID hold_bytes= 6,291,456 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= TRANS_CTX_MGR_ID hold_bytes= 3,694,592 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PLAN_CACHE_CTX_ID hold_bytes= 47,325,184 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 102,760,448 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= META_OBJ_CTX_ID hold_bytes= 33,341,440 limit= 214,748,360 [MEMORY] ctx_id= TX_CALLBACK_CTX_ID hold_bytes= 5,046,272 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= LOB_CTX_ID hold_bytes= 81,920 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= RPC_CTX_ID hold_bytes= 10,485,760 limit= 1,073,741,800 [MEMORY] ctx_id= TX_DATA_TABLE hold_bytes= 45,776,896 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MDS_DATA_ID hold_bytes= 2,228,224 limit= 9,223,372,036,854,775,807 [MEMORY][PM] tid= 63321 used= 0 hold= 2,097,152 pm=0x2b3a983d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 63325 used= 0 hold= 2,097,152 pm=0x2b3a987d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 65259 used= 0 hold= 4,194,304 pm=0x2b3abc1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 12536 used= 197,720 hold= 4,194,304 pm=0x2bcfc9bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 12807 used= 131,136 hold= 4,194,304 pm=0x2b3c17152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 27554 used= 336,024 hold= 4,194,304 pm=0x2bd048ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= summary used= 664,880 hold= 20,971,520 [2024-06-25 17:00:18.760017] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=3] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1003) [2024-06-25 17:00:18.760028] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=9] start lib cache evict(tenant_id=1003, mem_hold=230686720, mem_limit=4026531840, cache_obj_num=717, cache_node_num=717) [2024-06-25 17:00:18.760036] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=7] end lib cache evict(tenant_id=1003, cache_evict_num=0, mem_hold=230686720, mem_limit=4026531840, cache_obj_num=717, cache_node_num=717) [2024-06-25 17:00:18.760041] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=4] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:18.760060] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=6] [MEMORY] tenant_id= 1002 ctx_id= DEFAULT_CTX_ID hold= 548,995,072 used= 470,974,960 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 91 washed_blocks= 767 washed_size= 92,307,456 [MEMORY] hold= 209,817,600 used= 209,715,200 count= 5 avg_used= 41,943,040 block_cnt= 5 chunk_cnt= 5 mod=LogGroupBuffer [MEMORY] hold= 22,888,448 used= 22,877,184 count= 11 avg_used= 2,079,744 block_cnt= 11 chunk_cnt= 11 mod=CACHE_MAP_NODE [MEMORY] hold= 18,653,184 used= 16,689,651 count= 253 avg_used= 65,967 block_cnt= 253 chunk_cnt= 64 mod=SqlDtlBuf [MEMORY] hold= 18,616,320 used= 18,493,440 count= 10 avg_used= 1,849,344 block_cnt= 10 chunk_cnt= 8 mod=LogDIOAligned [MEMORY] hold= 15,777,792 used= 14,024,704 count= 214 avg_used= 65,536 block_cnt= 214 chunk_cnt= 51 mod=MinorMergeMgr [MEMORY] hold= 14,008,320 used= 12,508,080 count= 190 avg_used= 65,832 block_cnt= 190 chunk_cnt= 30 mod=[T]ObSessionDIB [MEMORY] hold= 13,303,808 used= 13,289,408 count= 8 avg_used= 1,661,176 block_cnt= 8 chunk_cnt= 8 mod=MysqlRequesReco [MEMORY] hold= 10,776,576 used= 10,735,904 count= 11 avg_used= 975,991 block_cnt= 11 chunk_cnt= 7 mod=IoControl [MEMORY] hold= 8,895,632 used= 8,810,249 count= 79 avg_used= 111,522 block_cnt= 25 chunk_cnt= 5 mod=OmtTenant [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=RCSrv [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=ArcFetchQueue [MEMORY] hold= 8,257,536 used= 8,249,344 count= 8 avg_used= 1,031,168 block_cnt= 8 chunk_cnt= 6 mod=Compressor [MEMORY] hold= 8,257,536 used= 8,240,064 count= 2 avg_used= 4,120,032 block_cnt= 2 chunk_cnt= 2 mod=FreeTbltStream [MEMORY] hold= 5,529,600 used= 4,915,200 count= 75 avg_used= 65,536 block_cnt= 75 chunk_cnt= 36 mod=MajorMergeMgr [MEMORY] hold= 5,120,000 used= 5,079,040 count= 5 avg_used= 1,015,808 block_cnt= 5 chunk_cnt= 3 mod=CommitVersions [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 10 mod=CkptDgnMemCU [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMem [MEMORY] hold= 3,680,096 used= 3,649,568 count= 471 avg_used= 7,748 block_cnt= 471 chunk_cnt= 79 mod=TLDecoderAlloc [MEMORY] hold= 3,416,064 used= 3,014,656 count= 49 avg_used= 61,523 block_cnt= 49 chunk_cnt= 21 mod=[T]char [MEMORY] hold= 3,145,728 used= 2,228,224 count= 128 avg_used= 17,408 block_cnt= 128 chunk_cnt= 6 mod=SqlDtlQueue [MEMORY] hold= 2,756,608 used= 2,720,005 count= 3 avg_used= 906,668 block_cnt= 3 chunk_cnt= 2 mod=SqlDtl1stBuf [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=XATimeWheel [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=TransTimeWheel [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 2,523,136 used= 2,505,184 count= 11 avg_used= 227,744 block_cnt= 11 chunk_cnt= 10 mod=SQLSessionInfo [MEMORY] hold= 2,486,272 used= 2,457,920 count= 2 avg_used= 1,228,960 block_cnt= 2 chunk_cnt= 2 mod=HashBuckDmId [MEMORY] hold= 2,437,120 used= 2,414,336 count= 2 avg_used= 1,207,168 block_cnt= 2 chunk_cnt= 2 mod=SqlPlanMon [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=MediumTabletMap [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmChe [MEMORY] hold= 2,375,680 used= 2,359,536 count= 2 avg_used= 1,179,768 block_cnt= 2 chunk_cnt= 2 mod=HashBuckLCSta [MEMORY] hold= 2,248,704 used= 2,228,224 count= 1 avg_used= 2,228,224 block_cnt= 1 chunk_cnt= 1 mod=LogIOCb [MEMORY] hold= 2,080,768 used= 2,079,744 count= 1 avg_used= 2,079,744 block_cnt= 1 chunk_cnt= 1 mod=LogReplayTask [MEMORY] hold= 1,925,120 used= 1,885,600 count= 5 avg_used= 377,120 block_cnt= 5 chunk_cnt= 2 mod=ClogGe [MEMORY] hold= 1,784,704 used= 1,746,772 count= 14 avg_used= 124,769 block_cnt= 14 chunk_cnt= 14 mod=CsSstableReader [MEMORY] hold= 1,777,664 used= 1,770,696 count= 1 avg_used= 1,770,696 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 1,720,320 used= 1,541,568 count= 42 avg_used= 36,704 block_cnt= 42 chunk_cnt= 21 mod=CommSysVarFac [MEMORY] hold= 1,507,328 used= 1,504,384 count= 23 avg_used= 65,408 block_cnt= 23 chunk_cnt= 15 mod=MemTblObj [MEMORY] hold= 1,376,256 used= 1,343,488 count= 4 avg_used= 335,872 block_cnt= 4 chunk_cnt= 1 mod=IOWorkerLQ [MEMORY] hold= 1,265,328 used= 1,253,464 count= 60 avg_used= 20,891 block_cnt= 60 chunk_cnt= 26 mod=TabletMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPlanCac [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMonMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=RewriteRuleMap [MEMORY] hold= 1,045,344 used= 710,016 count= 4,602 avg_used= 154 block_cnt= 17 chunk_cnt= 14 mod=TLDecoderCtx [MEMORY] hold= 957,088 used= 854,886 count= 42 avg_used= 20,354 block_cnt= 39 chunk_cnt= 26 mod=SqlSessiQuerSql [MEMORY] hold= 933,888 used= 931,072 count= 1 avg_used= 931,072 block_cnt= 1 chunk_cnt= 1 mod=ArcSenderQueue [MEMORY] hold= 884,736 used= 461,376 count= 54 avg_used= 8,544 block_cnt= 54 chunk_cnt= 9 mod=[T]ObDSActionAr [MEMORY] hold= 829,632 used= 804,864 count= 258 avg_used= 3,119 block_cnt= 130 chunk_cnt= 2 mod=LSMap [MEMORY] hold= 811,008 used= 720,896 count= 11 avg_used= 65,536 block_cnt= 11 chunk_cnt= 7 mod=SuspectInfoMgr [MEMORY] hold= 802,944 used= 786,432 count= 256 avg_used= 3,072 block_cnt= 128 chunk_cnt= 4 mod=TabletToLS [MEMORY] hold= 802,816 used= 800,000 count= 1 avg_used= 800,000 block_cnt= 1 chunk_cnt= 1 mod=SqlFltSpanRec [MEMORY] hold= 786,752 used= 524,352 count= 36 avg_used= 14,565 block_cnt= 34 chunk_cnt= 2 mod=LogAlloc [MEMORY] hold= 738,592 used= 732,448 count= 94 avg_used= 7,792 block_cnt= 94 chunk_cnt= 12 mod=CkptDgnMemNode [MEMORY] hold= 712,000 used= 706,304 count= 89 avg_used= 7,936 block_cnt= 89 chunk_cnt= 21 mod=SqlSession [MEMORY] hold= 663,552 used= 655,360 count= 1 avg_used= 655,360 block_cnt= 1 chunk_cnt= 1 mod=FetchLog [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 655,360 used= 649,216 count= 2 avg_used= 324,608 block_cnt= 2 chunk_cnt= 1 mod=ServerObjecPool [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DupTbLease [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTF [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTR [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DetectorTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MultiVersionGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=OBJLockGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=FrzTrigger [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MdsT [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=ElectTimer [MEMORY] hold= 614,400 used= 553,040 count= 10 avg_used= 55,304 block_cnt= 10 chunk_cnt= 4 mod=TabletIDSetBkt [MEMORY] hold= 606,208 used= 589,872 count= 2 avg_used= 294,936 block_cnt= 2 chunk_cnt= 1 mod=TmpBlockMap [MEMORY] hold= 598,016 used= 590,232 count= 1 avg_used= 590,232 block_cnt= 1 chunk_cnt= 1 mod=DagNetIdMap [MEMORY] hold= 589,824 used= 569,088 count= 6 avg_used= 94,848 block_cnt= 6 chunk_cnt= 5 mod=ResourceGroup [MEMORY] hold= 535,872 used= 527,424 count= 118 avg_used= 4,469 block_cnt= 112 chunk_cnt= 43 mod=WorkerMap [MEMORY] hold= 532,480 used= 531,072 count= 1 avg_used= 531,072 block_cnt= 1 chunk_cnt= 1 mod=TransService [MEMORY] hold= 523,776 used= 523,264 count= 8 avg_used= 65,408 block_cnt= 8 chunk_cnt= 7 mod=ResultSet [MEMORY] hold= 458,752 used= 457,856 count= 7 avg_used= 65,408 block_cnt= 7 chunk_cnt= 6 mod=MemTblMgrObj [MEMORY] hold= 450,560 used= 440,928 count= 3 avg_used= 146,976 block_cnt= 3 chunk_cnt= 3 mod=bf_queue [MEMORY] hold= 450,560 used= 422,240 count= 5 avg_used= 84,448 block_cnt= 5 chunk_cnt= 3 mod=LSSvr [MEMORY] hold= 441,952 used= 424,582 count= 5 avg_used= 84,916 block_cnt= 5 chunk_cnt= 4 mod=SqlMemMgr [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ApplySrv [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ReplaySrv [MEMORY] hold= 409,344 used= 399,744 count= 6 avg_used= 66,624 block_cnt= 6 chunk_cnt= 3 mod=SqlExecutor [MEMORY] hold= 403,200 used= 396,800 count= 100 avg_used= 3,968 block_cnt= 56 chunk_cnt= 14 mod=CkpDgn [MEMORY] hold= 392,640 used= 383,936 count= 7 avg_used= 54,848 block_cnt= 7 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 368,640 used= 344,064 count= 4 avg_used= 86,016 block_cnt= 4 chunk_cnt= 4 mod=DiskIteratorSto [MEMORY] hold= 336,000 used= 333,312 count= 42 avg_used= 7,936 block_cnt= 42 chunk_cnt= 20 mod=SqlSessiVarMap [MEMORY] hold= 328,000 used= 325,376 count= 41 avg_used= 7,936 block_cnt= 41 chunk_cnt= 15 mod=PlanVaIdx [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=DmlStatHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=ColUsagHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=TmpMap [MEMORY] hold= 262,144 used= 261,632 count= 4 avg_used= 65,408 block_cnt= 4 chunk_cnt= 4 mod=TmpPageCache [MEMORY] hold= 256,016 used= 253,952 count= 32 avg_used= 7,936 block_cnt= 32 chunk_cnt= 17 mod=QueryAllocator [MEMORY] hold= 245,760 used= 243,840 count= 5 avg_used= 48,768 block_cnt= 5 chunk_cnt= 3 mod=palf_env [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagMap [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagNetMap [MEMORY] hold= 200,704 used= 196,608 count= 64 avg_used= 3,072 block_cnt= 32 chunk_cnt= 4 mod=TxCtxMgr [MEMORY] hold= 196,608 used= 196,224 count= 3 avg_used= 65,408 block_cnt= 3 chunk_cnt= 2 mod=LockMemObj [MEMORY] hold= 165,680 used= 160,392 count= 20 avg_used= 8,019 block_cnt= 20 chunk_cnt= 12 mod=TabletStats [MEMORY] hold= 147,456 used= 139,264 count= 1 avg_used= 139,264 block_cnt= 1 chunk_cnt= 1 mod=RFLTaskQueue [MEMORY] hold= 124,992 used= 123,008 count= 31 avg_used= 3,968 block_cnt= 23 chunk_cnt= 10 mod=TinyAllocator [MEMORY] hold= 114,688 used= 106,496 count= 1 avg_used= 106,496 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlMgr [MEMORY] hold= 112,000 used= 111,104 count= 14 avg_used= 7,936 block_cnt= 14 chunk_cnt= 10 mod=ParamCharsType [MEMORY] hold= 106,496 used= 98,320 count= 1 avg_used= 98,320 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 106,496 used= 98,320 count= 1 avg_used= 98,320 block_cnt= 1 chunk_cnt= 1 mod=T3MBucket [MEMORY] hold= 102,336 used= 101,504 count= 13 avg_used= 7,808 block_cnt= 13 chunk_cnt= 9 mod=HashNodeLCSta [MEMORY] hold= 94,272 used= 93,504 count= 12 avg_used= 7,792 block_cnt= 12 chunk_cnt= 3 mod=CkptDgnCUNode [MEMORY] hold= 90,112 used= 82,048 count= 1 avg_used= 82,048 block_cnt= 1 chunk_cnt= 1 mod=MetaMemMgr [MEMORY] hold= 89,232 used= 88,528 count= 11 avg_used= 8,048 block_cnt= 11 chunk_cnt= 10 mod=[T]ObPerfEventR [MEMORY] hold= 88,128 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 7 mod=LSLocCache [MEMORY] hold= 81,920 used= 51,240 count= 5 avg_used= 10,248 block_cnt= 5 chunk_cnt= 3 mod=TabletSvrBucket [MEMORY] hold= 81,920 used= 68,480 count= 5 avg_used= 13,696 block_cnt= 5 chunk_cnt= 2 mod=LogReplayStatus [MEMORY] hold= 81,920 used= 57,920 count= 5 avg_used= 11,584 block_cnt= 5 chunk_cnt= 3 mod=LogApplyStatus [MEMORY] hold= 78,720 used= 78,080 count= 10 avg_used= 7,808 block_cnt= 10 chunk_cnt= 4 mod=TabletIDSetNode [MEMORY] hold= 73,728 used= 72,736 count= 1 avg_used= 72,736 block_cnt= 1 chunk_cnt= 1 mod=LogSharedQueueT [MEMORY] hold= 73,728 used= 66,176 count= 1 avg_used= 66,176 block_cnt= 1 chunk_cnt= 1 mod=Rpc [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=FlushLog [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxDataMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TmpBlockManager [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=StorMetaCacheIO [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=FlushMeta [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxCtxMemObj [MEMORY] hold= 59,904 used= 59,072 count= 13 avg_used= 4,544 block_cnt= 13 chunk_cnt= 11 mod=[T]ObTraceEvent [MEMORY] hold= 51,632 used= 24,328 count= 373 avg_used= 65 block_cnt= 89 chunk_cnt= 30 mod=Coro [MEMORY] hold= 48,832 used= 44,840 count= 2 avg_used= 22,420 block_cnt= 2 chunk_cnt= 2 mod=DynamicFactor [MEMORY] hold= 47,232 used= 46,848 count= 6 avg_used= 7,808 block_cnt= 6 chunk_cnt= 5 mod=HashNodePlanCac [MEMORY] hold= 47,232 used= 46,848 count= 6 avg_used= 7,808 block_cnt= 6 chunk_cnt= 5 mod=SuspectInfoNode [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=Autoincrement [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 1 mod=ObDMMDL [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 3 mod=MatchLsnMap [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 3 mod=MdsTableMgr [MEMORY] hold= 38,752 used= 38,016 count= 11 avg_used= 3,456 block_cnt= 11 chunk_cnt= 10 mod=CACHE_INST [MEMORY] hold= 32,768 used= 17,408 count= 2 avg_used= 8,704 block_cnt= 2 chunk_cnt= 2 mod=ObLogEXTTP [MEMORY] hold= 32,768 used= 24,688 count= 2 avg_used= 12,344 block_cnt= 2 chunk_cnt= 2 mod=TLD_ClientTask [MEMORY] hold= 32,768 used= 25,600 count= 1 avg_used= 25,600 block_cnt= 1 chunk_cnt= 1 mod=TSQLSessionMgr [MEMORY] hold= 29,760 used= 25,296 count= 62 avg_used= 408 block_cnt= 58 chunk_cnt= 27 mod=[T]ObTLDecoderC [MEMORY] hold= 25,024 used= 23,552 count= 23 avg_used= 1,024 block_cnt= 19 chunk_cnt= 12 mod=TGTimer [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 23,520 used= 23,160 count= 5 avg_used= 4,632 block_cnt= 5 chunk_cnt= 3 mod=TTLTaskBucket [MEMORY] hold= 21,008 used= 17,808 count= 42 avg_used= 424 block_cnt= 30 chunk_cnt= 16 mod=PackStateMap [MEMORY] hold= 20,880 used= 17,808 count= 42 avg_used= 424 block_cnt= 34 chunk_cnt= 18 mod=ContextsMap [MEMORY] hold= 20,880 used= 17,808 count= 42 avg_used= 424 block_cnt= 34 chunk_cnt= 18 mod=SequenceIdMap [MEMORY] hold= 20,864 used= 17,808 count= 42 avg_used= 424 block_cnt= 32 chunk_cnt= 17 mod=SequenceMap [MEMORY] hold= 18,656 used= 4,560 count= 190 avg_used= 24 block_cnt= 51 chunk_cnt= 26 mod=[T]MemoryContex [MEMORY] hold= 16,768 used= 8,448 count= 3 avg_used= 2,816 block_cnt= 2 chunk_cnt= 1 mod=BaseLogWriter [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_AssignedMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TblCtxIMgr [MEMORY] hold= 16,384 used= 8,448 count= 1 avg_used= 8,448 block_cnt= 1 chunk_cnt= 1 mod=PalfEnv [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TableCtxMgr [MEMORY] hold= 16,384 used= 10,248 count= 1 avg_used= 10,248 block_cnt= 1 chunk_cnt= 1 mod=TmpMemBlkMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=backupTaskSched [MEMORY] hold= 16,000 used= 15,872 count= 2 avg_used= 7,936 block_cnt= 2 chunk_cnt= 2 mod=ShareBlksMap [MEMORY] hold= 16,000 used= 15,872 count= 2 avg_used= 7,936 block_cnt= 2 chunk_cnt= 2 mod=LSIter [MEMORY] hold= 15,680 used= 15,360 count= 5 avg_used= 3,072 block_cnt= 5 chunk_cnt= 3 mod=LockMemtableMgr [MEMORY] hold= 10,240 used= 10,112 count= 2 avg_used= 5,056 block_cnt= 1 chunk_cnt= 1 mod=Scheduler [MEMORY] hold= 9,392 used= 9,256 count= 2 avg_used= 4,628 block_cnt= 2 chunk_cnt= 2 mod=StoLocCache [MEMORY] hold= 8,832 used= 8,064 count= 12 avg_used= 672 block_cnt= 10 chunk_cnt= 10 mod=[T]ObWarningBuf [MEMORY] hold= 8,640 used= 7,488 count= 18 avg_used= 416 block_cnt= 12 chunk_cnt= 7 mod=OMT_Worker [MEMORY] hold= 8,080 used= 7,720 count= 5 avg_used= 1,544 block_cnt= 5 chunk_cnt= 1 mod=TXBFRetryId [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LockWaitMgr [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LCLSender [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ls_id [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DASIDCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LsElectRefInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=APPLY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=REPLAY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MdsDebugMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=FetchLogTask [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=PurgeThrottle [MEMORY] hold= 7,872 used= 7,808 count= 1 avg_used= 7,808 block_cnt= 1 chunk_cnt= 1 mod=DirToBlk [MEMORY] hold= 7,856 used= 7,792 count= 1 avg_used= 7,792 block_cnt= 1 chunk_cnt= 1 mod=ColUsagNode [MEMORY] hold= 7,488 used= 7,424 count= 1 avg_used= 7,424 block_cnt= 1 chunk_cnt= 1 mod=LogIOWS [MEMORY] hold= 6,416 used= 6,216 count= 3 avg_used= 2,072 block_cnt= 2 chunk_cnt= 2 mod=DetectManager [MEMORY] hold= 5,456 used= 3,744 count= 26 avg_used= 144 block_cnt= 22 chunk_cnt= 8 mod=ObTxLSLogCb [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResRuleIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResGrpIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=PxPoolBkt [MEMORY] hold= 4,240 used= 3,880 count= 5 avg_used= 776 block_cnt= 4 chunk_cnt= 2 mod=DepTabletSet [MEMORY] hold= 4,160 used= 4,096 count= 1 avg_used= 4,096 block_cnt= 1 chunk_cnt= 1 mod=MacroFile [MEMORY] hold= 3,776 used= 3,712 count= 1 avg_used= 3,712 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlDfc [MEMORY] hold= 3,088 used= 3,016 count= 1 avg_used= 3,016 block_cnt= 1 chunk_cnt= 1 mod=HeapVar [MEMORY] hold= 2,768 used= 2,360 count= 5 avg_used= 472 block_cnt= 5 chunk_cnt= 4 mod=Election [MEMORY] hold= 2,352 used= 1,792 count= 8 avg_used= 224 block_cnt= 2 chunk_cnt= 1 mod=LogIOTask [MEMORY] hold= 2,272 used= 1,664 count= 8 avg_used= 208 block_cnt= 7 chunk_cnt= 7 mod=[T]ObMacroBlock [MEMORY] hold= 2,048 used= 1,984 count= 1 avg_used= 1,984 block_cnt= 1 chunk_cnt= 1 mod=CreateContext [MEMORY] hold= 1,920 used= 1,800 count= 1 avg_used= 1,800 block_cnt= 1 chunk_cnt= 1 mod=PxResMgr [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=MSTXCTX [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TabStatMgrLock [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=DUP_LS_SET [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GCMemtableMap [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 1,120 used= 1,056 count= 1 avg_used= 1,056 block_cnt= 1 chunk_cnt= 1 mod=LOG_HASH_MAP [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TLD_TIMER [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=CheckPointTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=FlushTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=DiskUsageTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TabShellTimer [MEMORY] hold= 1,072 used= 320 count= 10 avg_used= 32 block_cnt= 10 chunk_cnt= 6 mod=PalfFSCbNode [MEMORY] hold= 928 used= 520 count= 5 avg_used= 104 block_cnt= 5 chunk_cnt= 4 mod=RedoLogBuckLock [MEMORY] hold= 768 used= 640 count= 2 avg_used= 320 block_cnt= 1 chunk_cnt= 1 mod=PoolArenaArray [MEMORY] hold= 752 used= 648 count= 1 avg_used= 648 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfo [MEMORY] hold= 640 used= 160 count= 5 avg_used= 32 block_cnt= 5 chunk_cnt= 5 mod=RebuildCbNode [MEMORY] hold= 640 used= 16 count= 8 avg_used= 2 block_cnt= 8 chunk_cnt= 8 mod=[T]TSIUseWeak [MEMORY] hold= 624 used= 552 count= 1 avg_used= 552 block_cnt= 1 chunk_cnt= 1 mod=LSFreeze [MEMORY] hold= 592 used= 160 count= 5 avg_used= 32 block_cnt= 5 chunk_cnt= 5 mod=PalfRCCbNode [MEMORY] hold= 560 used= 205 count= 5 avg_used= 41 block_cnt= 5 chunk_cnt= 5 mod=KeepAliveBuf [MEMORY] hold= 480 used= 120 count= 5 avg_used= 24 block_cnt= 1 chunk_cnt= 1 mod=FreezeTask [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=HAScheduler [MEMORY] hold= 336 used= 144 count= 2 avg_used= 72 block_cnt= 2 chunk_cnt= 1 mod=TntSrvObjPool [MEMORY] hold= 176 used= 112 count= 1 avg_used= 112 block_cnt= 1 chunk_cnt= 1 mod=Coordinator [MEMORY] hold= 160 used= 93 count= 1 avg_used= 93 block_cnt= 1 chunk_cnt= 1 mod=TxLSLogBuf [MEMORY] hold= 160 used= 16 count= 2 avg_used= 8 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTHandler [MEMORY] hold= 128 used= 56 count= 1 avg_used= 56 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfoMap [MEMORY] hold= 80 used= 16 count= 1 avg_used= 16 block_cnt= 1 chunk_cnt= 1 mod=logservice [MEMORY] hold= 470,974,960 used= 460,645,803 count= 10,481 avg_used= 43,950 mod=SUMMARY [2024-06-25 17:00:18.760092] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=27] [MEMORY] tenant_id= 1002 ctx_id= MEMSTORE_CTX_ID hold= 4,194,304 used= 4,161,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 4,161,536 used= 4,159,632 count= 2 avg_used= 2,079,816 block_cnt= 2 chunk_cnt= 2 mod=Memstore [MEMORY] hold= 4,161,536 used= 4,159,632 count= 2 avg_used= 2,079,816 mod=SUMMARY [2024-06-25 17:00:18.760098] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1002 ctx_id= EXECUTE_CTX_ID hold= 6,291,456 used= 40,000 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 3 mod=SqlExecContext [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.760104] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1002 ctx_id= TRANS_CTX_MGR_ID hold= 3,694,592 used= 3,645,440 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 3 washed_blocks= 3 washed_size= 2,596,864 [MEMORY] hold= 3,645,440 used= 3,612,480 count= 5 avg_used= 722,496 block_cnt= 5 chunk_cnt= 3 mod=PartTranCtxMgr [MEMORY] hold= 3,645,440 used= 3,612,480 count= 5 avg_used= 722,496 mod=SUMMARY [2024-06-25 17:00:18.760114] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1002 ctx_id= PLAN_CACHE_CTX_ID hold= 47,325,184 used= 17,449,728 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 22 washed_blocks= 101 washed_size= 32,366,592 [MEMORY] hold= 13,820,672 used= 12,671,679 count= 579 avg_used= 21,885 block_cnt= 579 chunk_cnt= 36 mod=SqlPhyPlan [MEMORY] hold= 2,232,128 used= 2,214,144 count= 279 avg_used= 7,936 block_cnt= 279 chunk_cnt= 31 mod=OB_SQLSTAT_OBJ [MEMORY] hold= 798,848 used= 773,760 count= 390 avg_used= 1,984 block_cnt= 149 chunk_cnt= 31 mod=CreateContext [MEMORY] hold= 596,352 used= 557,416 count= 60 avg_used= 9,290 block_cnt= 60 chunk_cnt= 27 mod=SqlPlanCache [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=PlanCache [MEMORY] hold= 17,449,728 used= 16,218,599 count= 1,310 avg_used= 12,380 mod=SUMMARY [2024-06-25 17:00:18.760126] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1002 ctx_id= CO_STACK hold= 102,760,448 used= 98,058,240 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 98,058,240 used= 97,875,840 count= 190 avg_used= 515,136 block_cnt= 190 chunk_cnt= 49 mod=CoStack [MEMORY] hold= 98,058,240 used= 97,875,840 count= 190 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.760137] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1002 ctx_id= KVSTORE_CACHE_ID hold= 123,731,968 used= 123,731,968 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 123,731,968 used= 0 count= 59 avg_used= 0 block_cnt= 0 chunk_cnt= 0 mod=KvstorCacheMb [MEMORY] hold= 123,731,968 used= 0 count= 59 avg_used= 0 mod=SUMMARY [2024-06-25 17:00:18.760139] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABF4538-00061669548FCBE8-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301543, send_timestamp=1719305996458595) [2024-06-25 17:00:18.760146] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1002 ctx_id= META_OBJ_CTX_ID hold= 33,341,440 used= 33,062,720 limit= 214,748,360 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 17 washed_blocks= 17 washed_size= 2,310,144 [MEMORY] hold= 29,491,200 used= 26,236,400 count= 400 avg_used= 65,591 block_cnt= 400 chunk_cnt= 16 mod=L_TabletPool [MEMORY] hold= 3,481,600 used= 3,466,624 count= 53 avg_used= 65,408 block_cnt= 53 chunk_cnt= 4 mod=N_TabletPool [MEMORY] hold= 89,920 used= 84,768 count= 2 avg_used= 42,384 block_cnt= 2 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 33,062,720 used= 29,787,792 count= 455 avg_used= 65,467 mod=SUMMARY [2024-06-25 17:00:18.760155] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=6] [MEMORY] tenant_id= 1002 ctx_id= TX_CALLBACK_CTX_ID hold= 5,046,272 used= 784,128 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 3 washed_blocks= 5 washed_size= 5,439,488 [MEMORY] hold= 784,128 used= 777,728 count= 98 avg_used= 7,936 block_cnt= 98 chunk_cnt= 5 mod=MemtableCallbac [MEMORY] hold= 784,128 used= 777,728 count= 98 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.760162] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1002 ctx_id= LOB_CTX_ID hold= 81,920 used= 65,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 1 washed_blocks= 1 washed_size= 2,015,232 [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LobAllocator [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 mod=SUMMARY [2024-06-25 17:00:18.760173] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1002 ctx_id= RPC_CTX_ID hold= 10,485,760 used= 425,984 limit= 1,073,741,800 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 294,912 used= 292,608 count= 18 avg_used= 16,256 block_cnt= 18 chunk_cnt= 1 mod=OB_WRS_CLUSTER_ [MEMORY] hold= 81,920 used= 81,280 count= 5 avg_used= 16,256 block_cnt= 5 chunk_cnt= 4 mod=OB_REMOTE_EXECU [MEMORY] hold= 32,768 used= 32,512 count= 2 avg_used= 16,256 block_cnt= 2 chunk_cnt= 1 mod=[L]OB_REMOTE_EX [MEMORY] hold= 16,384 used= 16,256 count= 1 avg_used= 16,256 block_cnt= 1 chunk_cnt= 1 mod=OB_REMOTE_SYNC_ [MEMORY] hold= 425,984 used= 422,656 count= 26 avg_used= 16,256 mod=SUMMARY [2024-06-25 17:00:18.760184] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1002 ctx_id= TX_DATA_TABLE hold= 45,776,896 used= 11,951,680 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 19 washed_blocks= 47 washed_size= 25,526,272 [MEMORY] hold= 10,607,680 used= 10,525,760 count= 15 avg_used= 701,717 block_cnt= 15 chunk_cnt= 11 mod=MEMTABLE_ARENA [MEMORY] hold= 1,304,000 used= 1,293,568 count= 163 avg_used= 7,936 block_cnt= 163 chunk_cnt= 25 mod=TX_DATA_SLICE [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 1 mod=TX_DATA_ARENA [MEMORY] hold= 11,951,680 used= 11,859,008 count= 183 avg_used= 64,803 mod=SUMMARY [2024-06-25 17:00:18.760194] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=6] [MEMORY] tenant_id= 1002 ctx_id= MDS_DATA_ID hold= 2,228,224 used= 56,000 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 1 washed_blocks= 3 washed_size= 1,966,080 [MEMORY] hold= 56,000 used= 55,552 count= 7 avg_used= 7,936 block_cnt= 7 chunk_cnt= 2 mod=MdsTable [MEMORY] hold= 56,000 used= 55,552 count= 7 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.760217] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant: 1003, limit: 80,530,636,800 hold: 845,557,760 rpc_hold: 0 cache_hold: 12,582,912 cache_used: 12,582,912 cache_item_count: 6 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 461,778,944 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PLAN_CACHE_CTX_ID hold_bytes= 230,686,720 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= GLIBC hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 132,120,576 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= META_OBJ_CTX_ID hold_bytes= 2,097,152 limit= 16,106,127,360 [MEMORY] ctx_id= LOB_CTX_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY][PM] tid= 100566 used= 0 hold= 4,194,304 pm=0x2b3b55352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100567 used= 0 hold= 4,194,304 pm=0x2b3b553d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100574 used= 0 hold= 4,194,304 pm=0x2b3b55752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100575 used= 0 hold= 4,194,304 pm=0x2b3b557d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100576 used= 0 hold= 4,194,304 pm=0x2b3b55856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= summary used= 0 hold= 20,971,520 [2024-06-25 17:00:18.760319] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1003 ctx_id= DEFAULT_CTX_ID hold= 461,778,944 used= 395,343,200 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 196,411,392 used= 196,306,880 count= 96 avg_used= 2,044,863 block_cnt= 96 chunk_cnt= 96 mod=MysqlRequesReco [MEMORY] hold= 26,009,600 used= 25,968,640 count= 2 avg_used= 12,984,320 block_cnt= 2 chunk_cnt= 2 mod=ServerObjecPool [MEMORY] hold= 18,284,544 used= 16,326,336 count= 248 avg_used= 65,832 block_cnt= 248 chunk_cnt= 24 mod=[T]ObSessionDIB [MEMORY] hold= 12,603,392 used= 12,582,952 count= 1 avg_used= 12,582,952 block_cnt= 1 chunk_cnt= 1 mod=TabletMap [MEMORY] hold= 10,711,040 used= 10,670,496 count= 10 avg_used= 1,067,049 block_cnt= 10 chunk_cnt= 6 mod=IoControl [MEMORY] hold= 8,895,632 used= 8,810,249 count= 79 avg_used= 111,522 block_cnt= 25 chunk_cnt= 5 mod=OmtTenant [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=ArcFetchQueue [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=RCSrv [MEMORY] hold= 8,257,536 used= 8,240,064 count= 2 avg_used= 4,120,032 block_cnt= 2 chunk_cnt= 2 mod=FreeTbltStream [MEMORY] hold= 8,151,040 used= 8,131,072 count= 1 avg_used= 8,131,072 block_cnt= 1 chunk_cnt= 1 mod=TransService [MEMORY] hold= 6,463,808 used= 6,364,368 count= 1,374 avg_used= 4,632 block_cnt= 1,374 chunk_cnt= 36 mod=WorkerMap [MEMORY] hold= 6,340,608 used= 5,673,162 count= 86 avg_used= 65,967 block_cnt= 86 chunk_cnt= 5 mod=SqlDtlBuf [MEMORY] hold= 4,739,072 used= 4,718,712 count= 1 avg_used= 4,718,712 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMonMap [MEMORY] hold= 4,161,536 used= 4,159,488 count= 2 avg_used= 2,079,744 block_cnt= 2 chunk_cnt= 2 mod=CACHE_MAP_NODE [MEMORY] hold= 4,055,040 used= 3,588,096 count= 57 avg_used= 62,949 block_cnt= 57 chunk_cnt= 14 mod=[T]char [MEMORY] hold= 3,944,768 used= 3,912,448 count= 491 avg_used= 7,968 block_cnt= 491 chunk_cnt= 29 mod=ParamCharsType [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMem [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMemCU [MEMORY] hold= 3,145,728 used= 2,228,224 count= 128 avg_used= 17,408 block_cnt= 128 chunk_cnt= 6 mod=SqlDtlQueue [MEMORY] hold= 2,756,608 used= 2,720,005 count= 3 avg_used= 906,668 block_cnt= 3 chunk_cnt= 3 mod=SqlDtl1stBuf [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=TransTimeWheel [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=XATimeWheel [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=MediumTabletMap [MEMORY] hold= 2,375,680 used= 2,359,536 count= 2 avg_used= 1,179,768 block_cnt= 2 chunk_cnt= 2 mod=HashBuckLCSta [MEMORY] hold= 2,293,760 used= 2,277,440 count= 10 avg_used= 227,744 block_cnt= 10 chunk_cnt= 5 mod=SQLSessionInfo [MEMORY] hold= 2,248,704 used= 2,228,224 count= 1 avg_used= 2,228,224 block_cnt= 1 chunk_cnt= 1 mod=LogIOCb [MEMORY] hold= 1,777,664 used= 1,770,696 count= 1 avg_used= 1,770,696 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 1,670,592 used= 1,663,936 count= 7 avg_used= 237,705 block_cnt= 7 chunk_cnt= 2 mod=PoolFreeList [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=T3MBucket [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=RewriteRuleMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPlanCac [MEMORY] hold= 958,464 used= 950,272 count= 1 avg_used= 950,272 block_cnt= 1 chunk_cnt= 1 mod=IOWorkerLQ [MEMORY] hold= 933,888 used= 931,072 count= 1 avg_used= 931,072 block_cnt= 1 chunk_cnt= 1 mod=ArcSenderQueue [MEMORY] hold= 829,632 used= 804,864 count= 258 avg_used= 3,119 block_cnt= 130 chunk_cnt= 3 mod=LSMap [MEMORY] hold= 802,944 used= 786,432 count= 256 avg_used= 3,072 block_cnt= 128 chunk_cnt= 3 mod=TabletToLS [MEMORY] hold= 802,816 used= 800,000 count= 1 avg_used= 800,000 block_cnt= 1 chunk_cnt= 1 mod=SqlFltSpanRec [MEMORY] hold= 802,816 used= 800,000 count= 1 avg_used= 800,000 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMon [MEMORY] hold= 786,560 used= 524,352 count= 33 avg_used= 15,889 block_cnt= 33 chunk_cnt= 3 mod=LogAlloc [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 663,552 used= 655,360 count= 1 avg_used= 655,360 block_cnt= 1 chunk_cnt= 1 mod=FetchLog [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DupTbLease [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=FrzTrigger [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DetectorTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=ElectTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTF [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTR [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MdsT [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=OBJLockGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MultiVersionGC [MEMORY] hold= 598,016 used= 590,232 count= 1 avg_used= 590,232 block_cnt= 1 chunk_cnt= 1 mod=DagNetIdMap [MEMORY] hold= 434,176 used= 416,870 count= 4 avg_used= 104,217 block_cnt= 4 chunk_cnt= 3 mod=SqlMemMgr [MEMORY] hold= 432,000 used= 428,544 count= 54 avg_used= 7,936 block_cnt= 54 chunk_cnt= 22 mod=NotParamInfo [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ReplaySrv [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ApplySrv [MEMORY] hold= 385,024 used= 375,520 count= 2 avg_used= 187,760 block_cnt= 2 chunk_cnt= 1 mod=bf_queue [MEMORY] hold= 348,976 used= 338,016 count= 42 avg_used= 8,048 block_cnt= 42 chunk_cnt= 4 mod=[T]ObPerfEventR [MEMORY] hold= 319,488 used= 307,280 count= 2 avg_used= 153,640 block_cnt= 2 chunk_cnt= 2 mod=HashBuckDmId [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=ColUsagHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=DmlStatHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmChe [MEMORY] hold= 232,000 used= 230,144 count= 29 avg_used= 7,936 block_cnt= 29 chunk_cnt= 14 mod=LSIter [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagNetMap [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagMap [MEMORY] hold= 200,704 used= 196,608 count= 64 avg_used= 3,072 block_cnt= 32 chunk_cnt= 4 mod=TxCtxMgr [MEMORY] hold= 193,536 used= 190,848 count= 42 avg_used= 4,544 block_cnt= 42 chunk_cnt= 6 mod=[T]ObTraceEvent [MEMORY] hold= 147,456 used= 139,264 count= 1 avg_used= 139,264 block_cnt= 1 chunk_cnt= 1 mod=RFLTaskQueue [MEMORY] hold= 114,688 used= 106,496 count= 1 avg_used= 106,496 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlMgr [MEMORY] hold= 110,208 used= 109,312 count= 14 avg_used= 7,808 block_cnt= 14 chunk_cnt= 10 mod=HashNodeLCSta [MEMORY] hold= 98,304 used= 51,264 count= 6 avg_used= 8,544 block_cnt= 6 chunk_cnt= 5 mod=[T]ObDSActionAr [MEMORY] hold= 90,112 used= 82,048 count= 1 avg_used= 82,048 block_cnt= 1 chunk_cnt= 1 mod=MetaMemMgr [MEMORY] hold= 73,728 used= 72,736 count= 1 avg_used= 72,736 block_cnt= 1 chunk_cnt= 1 mod=LogSharedQueueT [MEMORY] hold= 64,640 used= 31,888 count= 446 avg_used= 71 block_cnt= 82 chunk_cnt= 20 mod=Coro [MEMORY] hold= 55,104 used= 54,656 count= 7 avg_used= 7,808 block_cnt= 7 chunk_cnt= 6 mod=HashNodePlanCac [MEMORY] hold= 48,832 used= 44,840 count= 2 avg_used= 22,420 block_cnt= 2 chunk_cnt= 2 mod=DynamicFactor [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=Autoincrement [MEMORY] hold= 32,768 used= 25,600 count= 1 avg_used= 25,600 block_cnt= 1 chunk_cnt= 1 mod=TSQLSessionMgr [MEMORY] hold= 32,768 used= 24,688 count= 2 avg_used= 12,344 block_cnt= 2 chunk_cnt= 2 mod=TLD_ClientTask [MEMORY] hold= 32,768 used= 17,408 count= 2 avg_used= 8,704 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTTP [MEMORY] hold= 30,912 used= 28,224 count= 42 avg_used= 672 block_cnt= 18 chunk_cnt= 6 mod=[T]ObWarningBuf [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 24,224 used= 5,952 count= 248 avg_used= 24 block_cnt= 68 chunk_cnt= 18 mod=[T]MemoryContex [MEMORY] hold= 21,760 used= 20,480 count= 20 avg_used= 1,024 block_cnt= 17 chunk_cnt= 12 mod=TGTimer [MEMORY] hold= 20,176 used= 17,472 count= 42 avg_used= 416 block_cnt= 22 chunk_cnt= 4 mod=OMT_Worker [MEMORY] hold= 16,768 used= 8,448 count= 3 avg_used= 2,816 block_cnt= 2 chunk_cnt= 2 mod=BaseLogWriter [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TblCtxIMgr [MEMORY] hold= 16,384 used= 8,448 count= 1 avg_used= 8,448 block_cnt= 1 chunk_cnt= 1 mod=PalfEnv [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TableCtxMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_AssignedMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=backupTaskSched [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TabletStats [MEMORY] hold= 8,192 used= 8,056 count= 2 avg_used= 4,028 block_cnt= 2 chunk_cnt= 2 mod=StoLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=APPLY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DASIDCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LsElectRefInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=REPLAY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ObDMMDL [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LSLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ShareBlksMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LCLSender [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LockWaitMgr [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MdsDebugMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 7,088 used= 6,912 count= 2 avg_used= 3,456 block_cnt= 2 chunk_cnt= 2 mod=CACHE_INST [MEMORY] hold= 6,656 used= 4,416 count= 35 avg_used= 126 block_cnt= 35 chunk_cnt= 17 mod=NotParamIdex [MEMORY] hold= 6,416 used= 6,216 count= 3 avg_used= 2,072 block_cnt= 2 chunk_cnt= 2 mod=DetectManager [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResRuleIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=PxPoolBkt [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResGrpIdMap [MEMORY] hold= 4,160 used= 4,096 count= 1 avg_used= 4,096 block_cnt= 1 chunk_cnt= 1 mod=MacroFile [MEMORY] hold= 3,776 used= 3,712 count= 1 avg_used= 3,712 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlDfc [MEMORY] hold= 3,328 used= 3,200 count= 2 avg_used= 1,600 block_cnt= 1 chunk_cnt= 1 mod=PoolArenaArray [MEMORY] hold= 2,304 used= 1,792 count= 8 avg_used= 224 block_cnt= 1 chunk_cnt= 1 mod=LogIOTask [MEMORY] hold= 1,920 used= 1,856 count= 1 avg_used= 1,856 block_cnt= 1 chunk_cnt= 1 mod=LogIOWS [MEMORY] hold= 1,920 used= 1,800 count= 1 avg_used= 1,800 block_cnt= 1 chunk_cnt= 1 mod=PxResMgr [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TabStatMgrLock [MEMORY] hold= 1,600 used= 1,344 count= 4 avg_used= 336 block_cnt= 4 chunk_cnt= 2 mod=NegParamIdex [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=DUP_LS_SET [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GCMemtableMap [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 1,120 used= 1,056 count= 1 avg_used= 1,056 block_cnt= 1 chunk_cnt= 1 mod=LOG_HASH_MAP [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=CheckPointTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TLD_TIMER [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TabShellTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=DiskUsageTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=FlushTimer [MEMORY] hold= 720 used= 648 count= 1 avg_used= 648 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfo [MEMORY] hold= 624 used= 552 count= 1 avg_used= 552 block_cnt= 1 chunk_cnt= 1 mod=LSFreeze [MEMORY] hold= 480 used= 120 count= 5 avg_used= 24 block_cnt= 1 chunk_cnt= 1 mod=FreezeTask [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=HAScheduler [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=Scheduler [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=MSTXCTX [MEMORY] hold= 336 used= 144 count= 2 avg_used= 72 block_cnt= 2 chunk_cnt= 1 mod=TntSrvObjPool [MEMORY] hold= 320 used= 8 count= 4 avg_used= 2 block_cnt= 4 chunk_cnt= 4 mod=[T]TSIUseWeak [MEMORY] hold= 192 used= 112 count= 1 avg_used= 112 block_cnt= 1 chunk_cnt= 1 mod=Coordinator [MEMORY] hold= 160 used= 16 count= 2 avg_used= 8 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTHandler [MEMORY] hold= 128 used= 56 count= 1 avg_used= 56 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfoMap [MEMORY] hold= 80 used= 16 count= 1 avg_used= 16 block_cnt= 1 chunk_cnt= 1 mod=logservice [MEMORY] hold= 395,343,200 used= 389,702,966 count= 5,983 avg_used= 65,135 mod=SUMMARY [2024-06-25 17:00:18.760352] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=19] [MEMORY] tenant_id= 1003 ctx_id= PLAN_CACHE_CTX_ID hold= 230,686,720 used= 218,693,232 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 204,144,528 used= 187,919,997 count= 9,180 avg_used= 20,470 block_cnt= 9,180 chunk_cnt= 109 mod=SqlPhyPlan [MEMORY] hold= 9,924,384 used= 9,145,448 count= 905 avg_used= 10,105 block_cnt= 905 chunk_cnt= 109 mod=SqlPlanCache [MEMORY] hold= 2,937,472 used= 2,845,056 count= 1,434 avg_used= 1,984 block_cnt= 361 chunk_cnt= 103 mod=CreateContext [MEMORY] hold= 1,237,120 used= 1,196,088 count= 140 avg_used= 8,543 block_cnt= 140 chunk_cnt= 2 mod=SqlPhyPlObj [MEMORY] hold= 448,000 used= 444,416 count= 56 avg_used= 7,936 block_cnt= 56 chunk_cnt= 18 mod=OB_SQLSTAT_OBJ [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=PlanCache [MEMORY] hold= 218,693,232 used= 201,552,605 count= 11,717 avg_used= 17,201 mod=SUMMARY [2024-06-25 17:00:18.760363] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1003 ctx_id= GLIBC hold= 4,194,304 used= 2,327,872 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 2,279,072 used= 1,819,125 count= 6,525 avg_used= 278 block_cnt= 288 chunk_cnt= 2 mod=PlJit [MEMORY] hold= 48,800 used= 45,810 count= 42 avg_used= 1,090 block_cnt= 9 chunk_cnt= 1 mod=PlCodeGen [MEMORY] hold= 2,327,872 used= 1,864,935 count= 6,567 avg_used= 283 mod=SUMMARY [2024-06-25 17:00:18.760371] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1003 ctx_id= CO_STACK hold= 132,120,576 used= 128,507,904 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 128,507,904 used= 128,268,864 count= 249 avg_used= 515,136 block_cnt= 249 chunk_cnt= 63 mod=CoStack [MEMORY] hold= 128,507,904 used= 128,268,864 count= 249 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.760383] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1003 ctx_id= KVSTORE_CACHE_ID hold= 12,582,912 used= 12,582,912 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 12,582,912 used= 0 count= 6 avg_used= 0 block_cnt= 0 chunk_cnt= 0 mod=KvstorCacheMb [MEMORY] hold= 12,582,912 used= 0 count= 6 avg_used= 0 mod=SUMMARY [2024-06-25 17:00:18.760390] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1003 ctx_id= META_OBJ_CTX_ID hold= 2,097,152 used= 401,408 limit= 16,106,127,360 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 401,408 used= 400,064 count= 2 avg_used= 200,032 block_cnt= 2 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 401,408 used= 400,064 count= 2 avg_used= 200,032 mod=SUMMARY [2024-06-25 17:00:18.760400] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1003 ctx_id= LOB_CTX_ID hold= 2,097,152 used= 65,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LobAllocator [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 mod=SUMMARY [2024-06-25 17:00:18.760645] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant: 1004, limit: 724,775,731,200 hold: 622,220,120,064 rpc_hold: 0 cache_hold: 541,409,148,928 cache_used: 541,409,148,928 cache_item_count: 258,164 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 20,837,236,736 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MEMSTORE_CTX_ID hold_bytes= 55,475,961,856 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= TRANS_CTX_MGR_ID hold_bytes= 6,291,456 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PLAN_CACHE_CTX_ID hold_bytes= 1,232,424,960 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= GLIBC hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 683,671,552 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= META_OBJ_CTX_ID hold_bytes= 165,675,008 limit= 144,955,146,240 [MEMORY] ctx_id= TX_CALLBACK_CTX_ID hold_bytes= 322,961,408 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= LOB_CTX_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PS_CACHE_CTX_ID hold_bytes= 6,291,456 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= TX_DATA_TABLE hold_bytes= 2,053,193,728 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MDS_DATA_ID hold_bytes= 20,971,520 limit= 9,223,372,036,854,775,807 [MEMORY][PM] tid= 99915 used= 0 hold= 2,097,152 pm=0x2b3b32656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99916 used= 0 hold= 4,194,304 pm=0x2b3b326d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99917 used= 0 hold= 4,194,304 pm=0x2b3b32752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99920 used= 0 hold= 2,097,152 pm=0x2b3b328d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99921 used= 0 hold= 4,194,304 pm=0x2b3b32952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99925 used= 0 hold= 4,194,304 pm=0x2b3b32d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99926 used= 0 hold= 4,194,304 pm=0x2b3b32dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99927 used= 0 hold= 4,194,304 pm=0x2b3b32e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99928 used= 0 hold= 4,194,304 pm=0x2b3b32ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99929 used= 0 hold= 4,194,304 pm=0x2b3b32f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99930 used= 0 hold= 4,194,304 pm=0x2b3b32fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99931 used= 0 hold= 4,194,304 pm=0x2b3b33056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99932 used= 0 hold= 4,194,304 pm=0x2b3b330d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99933 used= 0 hold= 4,194,304 pm=0x2b3b33152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99934 used= 0 hold= 4,194,304 pm=0x2b3b331d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99935 used= 0 hold= 4,194,304 pm=0x2b3b33256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99936 used= 0 hold= 4,194,304 pm=0x2b3b332d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99937 used= 0 hold= 4,194,304 pm=0x2b3b33352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99938 used= 0 hold= 4,194,304 pm=0x2b3b333d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99939 used= 0 hold= 4,194,304 pm=0x2b3b33456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99940 used= 0 hold= 4,194,304 pm=0x2b3b334d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99941 used= 0 hold= 4,194,304 pm=0x2b3b33552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99942 used= 0 hold= 4,194,304 pm=0x2b3b335d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99943 used= 0 hold= 4,194,304 pm=0x2b3b33656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99944 used= 0 hold= 4,194,304 pm=0x2b3b336d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99945 used= 0 hold= 4,194,304 pm=0x2b3b33752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99946 used= 0 hold= 4,194,304 pm=0x2b3b337d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99947 used= 0 hold= 4,194,304 pm=0x2b3b33856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99948 used= 0 hold= 4,194,304 pm=0x2b3b338d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99949 used= 0 hold= 4,194,304 pm=0x2b3b33952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99950 used= 0 hold= 4,194,304 pm=0x2b3b339d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99951 used= 0 hold= 4,194,304 pm=0x2b3b33a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99952 used= 0 hold= 4,194,304 pm=0x2b3b33ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99953 used= 0 hold= 4,194,304 pm=0x2b3b33b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99954 used= 0 hold= 4,194,304 pm=0x2b3b33bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99955 used= 0 hold= 4,194,304 pm=0x2b3b33e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99956 used= 0 hold= 4,194,304 pm=0x2b3b33ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99957 used= 0 hold= 4,194,304 pm=0x2b3b33f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99958 used= 0 hold= 4,194,304 pm=0x2b3b33fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99959 used= 0 hold= 4,194,304 pm=0x2b3b34456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99960 used= 0 hold= 4,194,304 pm=0x2b3b344d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99961 used= 0 hold= 4,194,304 pm=0x2b3b34552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99962 used= 0 hold= 4,194,304 pm=0x2b3b345d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99963 used= 0 hold= 4,194,304 pm=0x2b3b34656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99964 used= 0 hold= 4,194,304 pm=0x2b3b346d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99965 used= 0 hold= 4,194,304 pm=0x2b3b34752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99966 used= 0 hold= 4,194,304 pm=0x2b3b347d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99967 used= 0 hold= 4,194,304 pm=0x2b3b34856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99968 used= 0 hold= 4,194,304 pm=0x2b3b348d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99969 used= 0 hold= 4,194,304 pm=0x2b3b34952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99970 used= 0 hold= 4,194,304 pm=0x2b3b349d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99971 used= 0 hold= 4,194,304 pm=0x2b3b34a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99972 used= 0 hold= 4,194,304 pm=0x2b3b34ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99973 used= 0 hold= 4,194,304 pm=0x2b3b34b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99974 used= 0 hold= 4,194,304 pm=0x2b3b34bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99975 used= 0 hold= 4,194,304 pm=0x2b3b34c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99976 used= 0 hold= 4,194,304 pm=0x2b3b34cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99977 used= 0 hold= 4,194,304 pm=0x2b3b34d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99978 used= 0 hold= 4,194,304 pm=0x2b3b34dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99979 used= 0 hold= 4,194,304 pm=0x2b3b34e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99980 used= 0 hold= 4,194,304 pm=0x2b3b34ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99981 used= 0 hold= 4,194,304 pm=0x2b3b34f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99982 used= 0 hold= 4,194,304 pm=0x2b3b34fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99983 used= 0 hold= 4,194,304 pm=0x2b3b35056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99984 used= 0 hold= 4,194,304 pm=0x2b3b350d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99985 used= 0 hold= 4,194,304 pm=0x2b3b35152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99986 used= 0 hold= 4,194,304 pm=0x2b3b351d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99987 used= 0 hold= 4,194,304 pm=0x2b3b35256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99988 used= 0 hold= 4,194,304 pm=0x2b3b352d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99989 used= 0 hold= 4,194,304 pm=0x2b3b35352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99990 used= 0 hold= 4,194,304 pm=0x2b3b353d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99991 used= 0 hold= 4,194,304 pm=0x2b3b35456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99992 used= 0 hold= 4,194,304 pm=0x2b3b354d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99993 used= 0 hold= 4,194,304 pm=0x2b3b35552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99994 used= 0 hold= 4,194,304 pm=0x2b3b355d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99995 used= 0 hold= 4,194,304 pm=0x2b3b35656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99996 used= 0 hold= 4,194,304 pm=0x2b3b356d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99997 used= 0 hold= 4,194,304 pm=0x2b3b35752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99998 used= 0 hold= 4,194,304 pm=0x2b3b357d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99999 used= 0 hold= 4,194,304 pm=0x2b3b35856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100000 used= 0 hold= 4,194,304 pm=0x2b3b358d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100001 used= 0 hold= 4,194,304 pm=0x2b3b35952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100002 used= 0 hold= 4,194,304 pm=0x2b3b359d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100003 used= 0 hold= 4,194,304 pm=0x2b3b35a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100004 used= 0 hold= 4,194,304 pm=0x2b3b35ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100005 used= 0 hold= 4,194,304 pm=0x2b3b35b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100006 used= 0 hold= 4,194,304 pm=0x2b3b35bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100007 used= 0 hold= 4,194,304 pm=0x2b3b35c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100008 used= 0 hold= 4,194,304 pm=0x2b3b35cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100009 used= 0 hold= 4,194,304 pm=0x2b3b35d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100010 used= 0 hold= 4,194,304 pm=0x2b3b35dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100011 used= 0 hold= 4,194,304 pm=0x2b3b35e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100012 used= 0 hold= 4,194,304 pm=0x2b3b35ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100013 used= 0 hold= 4,194,304 pm=0x2b3b35f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100014 used= 0 hold= 4,194,304 pm=0x2b3b35fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100015 used= 0 hold= 4,194,304 pm=0x2b3b36056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100016 used= 0 hold= 4,194,304 pm=0x2b3b360d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100017 used= 0 hold= 4,194,304 pm=0x2b3b36152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100018 used= 0 hold= 4,194,304 pm=0x2b3b361d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100019 used= 0 hold= 4,194,304 pm=0x2b3b36256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100020 used= 0 hold= 4,194,304 pm=0x2b3b362d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100021 used= 0 hold= 4,194,304 pm=0x2b3b36352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100022 used= 0 hold= 4,194,304 pm=0x2b3b363d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100023 used= 0 hold= 4,194,304 pm=0x2b3b36456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100024 used= 0 hold= 4,194,304 pm=0x2b3b364d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100025 used= 0 hold= 4,194,304 pm=0x2b3b36552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100026 used= 0 hold= 4,194,304 pm=0x2b3b365d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100027 used= 0 hold= 4,194,304 pm=0x2b3b36656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100028 used= 0 hold= 4,194,304 pm=0x2b3b366d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100029 used= 0 hold= 4,194,304 pm=0x2b3b36752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100030 used= 0 hold= 4,194,304 pm=0x2b3b367d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100031 used= 0 hold= 4,194,304 pm=0x2b3b36856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100032 used= 0 hold= 4,194,304 pm=0x2b3b368d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100033 used= 0 hold= 4,194,304 pm=0x2b3b36952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100034 used= 0 hold= 4,194,304 pm=0x2b3b369d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100035 used= 0 hold= 4,194,304 pm=0x2b3b36a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100036 used= 0 hold= 4,194,304 pm=0x2b3b36ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100037 used= 0 hold= 4,194,304 pm=0x2b3b36b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100038 used= 0 hold= 4,194,304 pm=0x2b3b36bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100039 used= 0 hold= 4,194,304 pm=0x2b3b36e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100040 used= 0 hold= 4,194,304 pm=0x2b3b36ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100041 used= 0 hold= 4,194,304 pm=0x2b3b36f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100042 used= 0 hold= 4,194,304 pm=0x2b3b36fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100043 used= 0 hold= 4,194,304 pm=0x2b3b37056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100044 used= 0 hold= 4,194,304 pm=0x2b3b370d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100045 used= 0 hold= 4,194,304 pm=0x2b3b37152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100046 used= 0 hold= 4,194,304 pm=0x2b3b371d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100047 used= 0 hold= 4,194,304 pm=0x2b3b37256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100048 used= 0 hold= 4,194,304 pm=0x2b3b372d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100049 used= 0 hold= 4,194,304 pm=0x2b3b37352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100050 used= 0 hold= 4,194,304 pm=0x2b3b373d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100051 used= 0 hold= 4,194,304 pm=0x2b3b37856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100052 used= 0 hold= 4,194,304 pm=0x2b3b378d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100053 used= 0 hold= 4,194,304 pm=0x2b3b37952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100054 used= 0 hold= 4,194,304 pm=0x2b3b379d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100055 used= 0 hold= 4,194,304 pm=0x2b3b37a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100056 used= 0 hold= 4,194,304 pm=0x2b3b37ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100057 used= 0 hold= 4,194,304 pm=0x2b3b37b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100058 used= 0 hold= 4,194,304 pm=0x2b3b37bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100059 used= 0 hold= 4,194,304 pm=0x2b3b37c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100060 used= 0 hold= 4,194,304 pm=0x2b3b37cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100061 used= 0 hold= 4,194,304 pm=0x2b3b37d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100062 used= 0 hold= 4,194,304 pm=0x2b3b37dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100063 used= 0 hold= 4,194,304 pm=0x2b3b37e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100064 used= 0 hold= 4,194,304 pm=0x2b3b37ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100065 used= 0 hold= 4,194,304 pm=0x2b3b37f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100066 used= 0 hold= 4,194,304 pm=0x2b3b37fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100067 used= 0 hold= 4,194,304 pm=0x2b3b38056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100068 used= 0 hold= 4,194,304 pm=0x2b3b380d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100069 used= 0 hold= 4,194,304 pm=0x2b3b38152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100070 used= 0 hold= 4,194,304 pm=0x2b3b381d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100071 used= 0 hold= 4,194,304 pm=0x2b3b38256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100072 used= 0 hold= 4,194,304 pm=0x2b3b382d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100073 used= 0 hold= 4,194,304 pm=0x2b3b38352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100074 used= 0 hold= 4,194,304 pm=0x2b3b383d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100075 used= 0 hold= 4,194,304 pm=0x2b3b38656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100076 used= 0 hold= 4,194,304 pm=0x2b3b386d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100077 used= 0 hold= 4,194,304 pm=0x2b3b38752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100078 used= 0 hold= 4,194,304 pm=0x2b3b387d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100079 used= 0 hold= 4,194,304 pm=0x2b3b38856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100080 used= 0 hold= 4,194,304 pm=0x2b3b388d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100081 used= 0 hold= 4,194,304 pm=0x2b3b38952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100082 used= 0 hold= 4,194,304 pm=0x2b3b389d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100083 used= 0 hold= 4,194,304 pm=0x2b3b38a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100084 used= 0 hold= 4,194,304 pm=0x2b3b38ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100085 used= 0 hold= 4,194,304 pm=0x2b3b38b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100086 used= 0 hold= 4,194,304 pm=0x2b3b38bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100087 used= 0 hold= 4,194,304 pm=0x2b3b38c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100088 used= 0 hold= 4,194,304 pm=0x2b3b38cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100089 used= 0 hold= 4,194,304 pm=0x2b3b38d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100090 used= 0 hold= 4,194,304 pm=0x2b3b38dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100091 used= 0 hold= 4,194,304 pm=0x2b3b38e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100092 used= 0 hold= 4,194,304 pm=0x2b3b38ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100093 used= 0 hold= 4,194,304 pm=0x2b3b38f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100094 used= 0 hold= 4,194,304 pm=0x2b3b38fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100095 used= 0 hold= 4,194,304 pm=0x2b3b39056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100096 used= 0 hold= 4,194,304 pm=0x2b3b390d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100097 used= 0 hold= 4,194,304 pm=0x2b3b39152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100098 used= 0 hold= 4,194,304 pm=0x2b3b391d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100099 used= 0 hold= 4,194,304 pm=0x2b3b39256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100100 used= 0 hold= 4,194,304 pm=0x2b3b392d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100101 used= 0 hold= 4,194,304 pm=0x2b3b39352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100102 used= 0 hold= 4,194,304 pm=0x2b3b393d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100103 used= 0 hold= 4,194,304 pm=0x2b3b39456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100104 used= 0 hold= 4,194,304 pm=0x2b3b394d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100105 used= 0 hold= 4,194,304 pm=0x2b3b39552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100106 used= 0 hold= 4,194,304 pm=0x2b3b395d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100107 used= 0 hold= 4,194,304 pm=0x2b3b39656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100108 used= 0 hold= 4,194,304 pm=0x2b3b396d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100109 used= 0 hold= 4,194,304 pm=0x2b3b39752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100110 used= 0 hold= 4,194,304 pm=0x2b3b397d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100111 used= 0 hold= 4,194,304 pm=0x2b3b39856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100112 used= 0 hold= 4,194,304 pm=0x2b3b398d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100113 used= 0 hold= 4,194,304 pm=0x2b3b39952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100114 used= 0 hold= 4,194,304 pm=0x2b3b399d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100115 used= 0 hold= 4,194,304 pm=0x2b3b39a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100116 used= 0 hold= 4,194,304 pm=0x2b3b39ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100117 used= 0 hold= 4,194,304 pm=0x2b3b39b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100118 used= 0 hold= 4,194,304 pm=0x2b3b39bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100119 used= 0 hold= 4,194,304 pm=0x2b3b39c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100120 used= 0 hold= 4,194,304 pm=0x2b3b39cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100121 used= 0 hold= 4,194,304 pm=0x2b3b39d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100122 used= 0 hold= 4,194,304 pm=0x2b3b39dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100123 used= 0 hold= 4,194,304 pm=0x2b3b39e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100124 used= 0 hold= 4,194,304 pm=0x2b3b39ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100125 used= 0 hold= 4,194,304 pm=0x2b3b39f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100126 used= 0 hold= 4,194,304 pm=0x2b3b39fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100127 used= 0 hold= 4,194,304 pm=0x2b3b3a056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100128 used= 0 hold= 4,194,304 pm=0x2b3b3a0d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100129 used= 0 hold= 4,194,304 pm=0x2b3b3a152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100130 used= 0 hold= 4,194,304 pm=0x2b3b3a1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100131 used= 0 hold= 4,194,304 pm=0x2b3b3a256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100132 used= 0 hold= 4,194,304 pm=0x2b3b3a2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100133 used= 0 hold= 4,194,304 pm=0x2b3b3a352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100134 used= 0 hold= 4,194,304 pm=0x2b3b3a3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100135 used= 0 hold= 4,194,304 pm=0x2b3b3a456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100136 used= 0 hold= 4,194,304 pm=0x2b3b3a4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100137 used= 0 hold= 4,194,304 pm=0x2b3b3a552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100138 used= 0 hold= 4,194,304 pm=0x2b3b3a5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100139 used= 0 hold= 4,194,304 pm=0x2b3b3a656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100140 used= 0 hold= 4,194,304 pm=0x2b3b3a6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100141 used= 0 hold= 4,194,304 pm=0x2b3b3a752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100142 used= 0 hold= 4,194,304 pm=0x2b3b3a7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100143 used= 0 hold= 4,194,304 pm=0x2b3b3a856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100144 used= 0 hold= 4,194,304 pm=0x2b3b3a8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100145 used= 0 hold= 4,194,304 pm=0x2b3b3a952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100146 used= 0 hold= 4,194,304 pm=0x2b3b3a9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100147 used= 0 hold= 4,194,304 pm=0x2b3b3aa56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100148 used= 0 hold= 4,194,304 pm=0x2b3b3aad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100149 used= 0 hold= 4,194,304 pm=0x2b3b3ab52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100150 used= 0 hold= 4,194,304 pm=0x2b3b3abd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100151 used= 0 hold= 4,194,304 pm=0x2b3b3ac56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100152 used= 0 hold= 4,194,304 pm=0x2b3b3acd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100153 used= 0 hold= 4,194,304 pm=0x2b3b3ad52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100154 used= 0 hold= 4,194,304 pm=0x2b3b3add0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100155 used= 0 hold= 4,194,304 pm=0x2b3b3b056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100156 used= 0 hold= 4,194,304 pm=0x2b3b3b0d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100157 used= 0 hold= 4,194,304 pm=0x2b3b3b152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100158 used= 0 hold= 4,194,304 pm=0x2b3b3b1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100159 used= 0 hold= 4,194,304 pm=0x2b3b3b256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100160 used= 0 hold= 4,194,304 pm=0x2b3b3b2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100161 used= 0 hold= 4,194,304 pm=0x2b3b3b352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100162 used= 0 hold= 4,194,304 pm=0x2b3b3b3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100163 used= 0 hold= 4,194,304 pm=0x2b3b3b656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100164 used= 0 hold= 4,194,304 pm=0x2b3b3b6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100165 used= 0 hold= 4,194,304 pm=0x2b3b3b752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100166 used= 0 hold= 4,194,304 pm=0x2b3b3b7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100167 used= 0 hold= 4,194,304 pm=0x2b3b3b856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100168 used= 0 hold= 4,194,304 pm=0x2b3b3b8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100169 used= 0 hold= 4,194,304 pm=0x2b3b3b952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100170 used= 0 hold= 4,194,304 pm=0x2b3b3b9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100171 used= 0 hold= 4,194,304 pm=0x2b3b3ba56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100172 used= 0 hold= 4,194,304 pm=0x2b3b3bad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100173 used= 0 hold= 4,194,304 pm=0x2b3b3bb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100174 used= 0 hold= 4,194,304 pm=0x2b3b3bbd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100175 used= 0 hold= 4,194,304 pm=0x2b3b3bc56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100176 used= 0 hold= 4,194,304 pm=0x2b3b3bcd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100177 used= 0 hold= 4,194,304 pm=0x2b3b3bd52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100178 used= 0 hold= 4,194,304 pm=0x2b3b3bdd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100179 used= 0 hold= 4,194,304 pm=0x2b3b3be56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100180 used= 0 hold= 4,194,304 pm=0x2b3b3bed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100181 used= 0 hold= 4,194,304 pm=0x2b3b3bf52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100182 used= 0 hold= 4,194,304 pm=0x2b3b3bfd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100183 used= 0 hold= 4,194,304 pm=0x2b3b3c056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100184 used= 0 hold= 4,194,304 pm=0x2b3b3c0d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100185 used= 0 hold= 4,194,304 pm=0x2b3b3c152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100186 used= 0 hold= 4,194,304 pm=0x2b3b3c1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100187 used= 0 hold= 4,194,304 pm=0x2b3b3c256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100188 used= 0 hold= 4,194,304 pm=0x2b3b3c2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100189 used= 0 hold= 4,194,304 pm=0x2b3b3c352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100190 used= 0 hold= 4,194,304 pm=0x2b3b3c3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100191 used= 0 hold= 4,194,304 pm=0x2b3b3c456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100192 used= 0 hold= 4,194,304 pm=0x2b3b3c4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100193 used= 0 hold= 4,194,304 pm=0x2b3b3c552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100194 used= 0 hold= 4,194,304 pm=0x2b3b3c5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100195 used= 0 hold= 4,194,304 pm=0x2b3b3c656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100196 used= 0 hold= 4,194,304 pm=0x2b3b3c6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100197 used= 0 hold= 4,194,304 pm=0x2b3b3c752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100198 used= 0 hold= 4,194,304 pm=0x2b3b3c7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100199 used= 0 hold= 4,194,304 pm=0x2b3b3c856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100200 used= 0 hold= 4,194,304 pm=0x2b3b3c8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100201 used= 0 hold= 4,194,304 pm=0x2b3b3c952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100202 used= 0 hold= 4,194,304 pm=0x2b3b3c9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100203 used= 0 hold= 4,194,304 pm=0x2b3b3ca56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100204 used= 0 hold= 4,194,304 pm=0x2b3b3cad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100205 used= 0 hold= 4,194,304 pm=0x2b3b3cb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100206 used= 0 hold= 4,194,304 pm=0x2b3b3cbd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100207 used= 0 hold= 4,194,304 pm=0x2b3b3cc56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100208 used= 0 hold= 4,194,304 pm=0x2b3b3ccd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100209 used= 0 hold= 4,194,304 pm=0x2b3b3cd52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100210 used= 0 hold= 4,194,304 pm=0x2b3b3cdd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100211 used= 0 hold= 4,194,304 pm=0x2b3b3d056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100212 used= 0 hold= 4,194,304 pm=0x2b3b3d0d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100213 used= 0 hold= 4,194,304 pm=0x2b3b3d152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100214 used= 0 hold= 4,194,304 pm=0x2b3b3d1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100215 used= 0 hold= 4,194,304 pm=0x2b3b3d256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100216 used= 0 hold= 4,194,304 pm=0x2b3b3d2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100217 used= 0 hold= 4,194,304 pm=0x2b3b3d352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100218 used= 0 hold= 4,194,304 pm=0x2b3b3d3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100219 used= 0 hold= 4,194,304 pm=0x2b3b3d456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100220 used= 0 hold= 4,194,304 pm=0x2b3b3d4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100221 used= 0 hold= 4,194,304 pm=0x2b3b3d552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100222 used= 0 hold= 4,194,304 pm=0x2b3b3d5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100223 used= 0 hold= 4,194,304 pm=0x2b3b3d656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100224 used= 0 hold= 4,194,304 pm=0x2b3b3d6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100225 used= 0 hold= 4,194,304 pm=0x2b3b3d752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100226 used= 0 hold= 4,194,304 pm=0x2b3b3d7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100227 used= 0 hold= 4,194,304 pm=0x2b3b3da56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100228 used= 0 hold= 4,194,304 pm=0x2b3b3dad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100229 used= 0 hold= 4,194,304 pm=0x2b3b3db52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100230 used= 0 hold= 4,194,304 pm=0x2b3b3dbd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100231 used= 0 hold= 4,194,304 pm=0x2b3b3dc56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100232 used= 0 hold= 4,194,304 pm=0x2b3b3dcd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100233 used= 0 hold= 4,194,304 pm=0x2b3b3dd52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100234 used= 0 hold= 4,194,304 pm=0x2b3b3ddd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100235 used= 0 hold= 4,194,304 pm=0x2b3b3de56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100236 used= 0 hold= 4,194,304 pm=0x2b3b3ded4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100237 used= 0 hold= 4,194,304 pm=0x2b3b3df52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100238 used= 0 hold= 4,194,304 pm=0x2b3b3dfd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100239 used= 0 hold= 4,194,304 pm=0x2b3b3e256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100240 used= 0 hold= 4,194,304 pm=0x2b3b3e2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100241 used= 0 hold= 4,194,304 pm=0x2b3b3e352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100242 used= 0 hold= 4,194,304 pm=0x2b3b3e3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100243 used= 0 hold= 4,194,304 pm=0x2b3b3e456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100244 used= 0 hold= 4,194,304 pm=0x2b3b3e4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100245 used= 0 hold= 4,194,304 pm=0x2b3b3e552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100246 used= 0 hold= 4,194,304 pm=0x2b3b3e5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100247 used= 0 hold= 4,194,304 pm=0x2b3b3e856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100248 used= 0 hold= 4,194,304 pm=0x2b3b3e8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100249 used= 0 hold= 4,194,304 pm=0x2b3b3e952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100250 used= 0 hold= 4,194,304 pm=0x2b3b3e9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100251 used= 0 hold= 4,194,304 pm=0x2b3b3ea56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100252 used= 0 hold= 4,194,304 pm=0x2b3b3ead4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100253 used= 0 hold= 4,194,304 pm=0x2b3b3eb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100254 used= 0 hold= 4,194,304 pm=0x2b3b3ebd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100255 used= 0 hold= 4,194,304 pm=0x2b3b3ec56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100256 used= 0 hold= 4,194,304 pm=0x2b3b3ecd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100257 used= 0 hold= 4,194,304 pm=0x2b3b3ed52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100258 used= 0 hold= 4,194,304 pm=0x2b3b3edd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100259 used= 0 hold= 4,194,304 pm=0x2b3b3ee56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100260 used= 0 hold= 4,194,304 pm=0x2b3b3eed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100261 used= 0 hold= 4,194,304 pm=0x2b3b3ef52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100262 used= 0 hold= 4,194,304 pm=0x2b3b3efd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100263 used= 0 hold= 4,194,304 pm=0x2b3b3f056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100264 used= 0 hold= 4,194,304 pm=0x2b3b3f0d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100265 used= 0 hold= 4,194,304 pm=0x2b3b3f152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100266 used= 0 hold= 4,194,304 pm=0x2b3b3f1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100267 used= 0 hold= 4,194,304 pm=0x2b3b3f256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100268 used= 0 hold= 4,194,304 pm=0x2b3b3f2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100269 used= 0 hold= 4,194,304 pm=0x2b3b3f352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100270 used= 0 hold= 4,194,304 pm=0x2b3b3f3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100271 used= 0 hold= 4,194,304 pm=0x2b3b3f456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100272 used= 0 hold= 4,194,304 pm=0x2b3b3f4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100273 used= 0 hold= 4,194,304 pm=0x2b3b3f552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100274 used= 0 hold= 4,194,304 pm=0x2b3b3f5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100275 used= 0 hold= 4,194,304 pm=0x2b3b3f656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100276 used= 0 hold= 4,194,304 pm=0x2b3b3f6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100277 used= 0 hold= 4,194,304 pm=0x2b3b3f752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100278 used= 0 hold= 4,194,304 pm=0x2b3b3f7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100279 used= 0 hold= 4,194,304 pm=0x2b3b3f856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100280 used= 0 hold= 4,194,304 pm=0x2b3b3f8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100281 used= 0 hold= 4,194,304 pm=0x2b3b3f952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100282 used= 0 hold= 4,194,304 pm=0x2b3b3f9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100283 used= 0 hold= 4,194,304 pm=0x2b3b3fa56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100284 used= 0 hold= 4,194,304 pm=0x2b3b3fad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100285 used= 0 hold= 4,194,304 pm=0x2b3b3fb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100286 used= 0 hold= 4,194,304 pm=0x2b3b3fbd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100287 used= 0 hold= 4,194,304 pm=0x2b3b3fc56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100288 used= 0 hold= 4,194,304 pm=0x2b3b3fcd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100289 used= 0 hold= 4,194,304 pm=0x2b3b3fd52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100290 used= 0 hold= 4,194,304 pm=0x2b3b3fdd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100291 used= 0 hold= 4,194,304 pm=0x2b3b40056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100292 used= 0 hold= 4,194,304 pm=0x2b3b400d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100293 used= 0 hold= 4,194,304 pm=0x2b3b40152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100294 used= 0 hold= 4,194,304 pm=0x2b3b401d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100295 used= 0 hold= 4,194,304 pm=0x2b3b40256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100296 used= 0 hold= 4,194,304 pm=0x2b3b402d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100297 used= 0 hold= 4,194,304 pm=0x2b3b40352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100298 used= 0 hold= 4,194,304 pm=0x2b3b403d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100299 used= 0 hold= 4,194,304 pm=0x2b3b40456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100300 used= 0 hold= 4,194,304 pm=0x2b3b404d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100301 used= 0 hold= 4,194,304 pm=0x2b3b40552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100302 used= 0 hold= 4,194,304 pm=0x2b3b405d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100303 used= 0 hold= 4,194,304 pm=0x2b3b40856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100304 used= 0 hold= 4,194,304 pm=0x2b3b408d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100305 used= 0 hold= 4,194,304 pm=0x2b3b40952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100306 used= 0 hold= 4,194,304 pm=0x2b3b409d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100307 used= 0 hold= 4,194,304 pm=0x2b3b40a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100308 used= 0 hold= 4,194,304 pm=0x2b3b40ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100309 used= 0 hold= 4,194,304 pm=0x2b3b40b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100310 used= 0 hold= 4,194,304 pm=0x2b3b40bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100311 used= 0 hold= 4,194,304 pm=0x2b3b40c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100312 used= 0 hold= 4,194,304 pm=0x2b3b40cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100313 used= 0 hold= 4,194,304 pm=0x2b3b40d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100314 used= 0 hold= 4,194,304 pm=0x2b3b40dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100315 used= 0 hold= 4,194,304 pm=0x2b3b40e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100316 used= 0 hold= 4,194,304 pm=0x2b3b40ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100317 used= 0 hold= 4,194,304 pm=0x2b3b40f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100318 used= 0 hold= 4,194,304 pm=0x2b3b40fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100319 used= 0 hold= 4,194,304 pm=0x2b3b41056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100320 used= 0 hold= 4,194,304 pm=0x2b3b410d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100321 used= 0 hold= 4,194,304 pm=0x2b3b41152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100322 used= 0 hold= 4,194,304 pm=0x2b3b411d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100323 used= 0 hold= 4,194,304 pm=0x2b3b41256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100324 used= 0 hold= 4,194,304 pm=0x2b3b412d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= summary used= 0 hold= 1,694,498,816 [2024-06-25 17:00:18.760839] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3944143, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=202, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:18.760834] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=39] [MEMORY] tenant_id= 1004 ctx_id= DEFAULT_CTX_ID hold= 20,837,236,736 used= 18,442,559,472 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 15,236,349,952 used= 15,228,866,240 count= 7,286 avg_used= 2,090,154 block_cnt= 7,286 chunk_cnt= 7,286 mod=MysqlRequesReco [MEMORY] hold= 2,014,183,424 used= 2,013,192,192 count= 968 avg_used= 2,079,744 block_cnt= 968 chunk_cnt= 968 mod=CACHE_MAP_NODE [MEMORY] hold= 209,817,600 used= 209,715,200 count= 5 avg_used= 41,943,040 block_cnt= 5 chunk_cnt= 5 mod=LogGroupBuffer [MEMORY] hold= 186,826,752 used= 166,068,224 count= 2,534 avg_used= 65,536 block_cnt= 2,534 chunk_cnt= 426 mod=MinorMergeMgr [MEMORY] hold= 95,846,400 used= 85,581,600 count= 1,300 avg_used= 65,832 block_cnt= 1,300 chunk_cnt= 68 mod=[T]ObSessionDIB [MEMORY] hold= 83,140,608 used= 83,099,648 count= 2 avg_used= 41,549,824 block_cnt= 2 chunk_cnt= 2 mod=ServerObjecPool [MEMORY] hold= 69,648,384 used= 59,359,232 count= 1,256 avg_used= 47,260 block_cnt= 1,256 chunk_cnt= 242 mod=[T]char [MEMORY] hold= 47,259,648 used= 42,008,576 count= 641 avg_used= 65,536 block_cnt= 641 chunk_cnt= 294 mod=MajorMergeMgr [MEMORY] hold= 39,519,904 used= 39,194,896 count= 5,032 avg_used= 7,789 block_cnt= 5,032 chunk_cnt= 399 mod=TLDecoderAlloc [MEMORY] hold= 39,350,272 used= 39,321,760 count= 2 avg_used= 19,660,880 block_cnt= 2 chunk_cnt= 2 mod=HashBuckDmId [MEMORY] hold= 38,772,736 used= 38,401,280 count= 19 avg_used= 2,021,120 block_cnt= 19 chunk_cnt= 19 mod=SqlPlanMon [MEMORY] hold= 37,769,216 used= 37,748,856 count= 1 avg_used= 37,748,856 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmChe [MEMORY] hold= 27,650,992 used= 27,222,528 count= 5,906 avg_used= 4,609 block_cnt= 5,864 chunk_cnt= 436 mod=WorkerMap [MEMORY] hold= 21,544,480 used= 11,700,168 count= 135,669 avg_used= 86 block_cnt= 331 chunk_cnt= 190 mod=TLDecoderCtx [MEMORY] hold= 19,169,280 used= 17,151,420 count= 260 avg_used= 65,967 block_cnt= 260 chunk_cnt= 65 mod=SqlDtlBuf [MEMORY] hold= 18,616,320 used= 18,493,440 count= 10 avg_used= 1,849,344 block_cnt= 10 chunk_cnt= 9 mod=LogDIOAligned [MEMORY] hold= 18,046,336 used= 17,863,936 count= 2,027 avg_used= 8,812 block_cnt= 2,027 chunk_cnt= 352 mod=ParamCharsType [MEMORY] hold= 16,924,288 used= 16,783,936 count= 10 avg_used= 1,678,393 block_cnt= 10 chunk_cnt= 10 mod=CsSstableReader [MEMORY] hold= 15,524,320 used= 15,480,328 count= 367 avg_used= 42,180 block_cnt= 367 chunk_cnt= 124 mod=TabletMap [MEMORY] hold= 13,064,768 used= 12,959,488 count= 1,633 avg_used= 7,936 block_cnt= 1,633 chunk_cnt= 311 mod=QueryAllocator [MEMORY] hold= 12,816,384 used= 12,763,552 count= 42 avg_used= 303,894 block_cnt= 42 chunk_cnt= 38 mod=IoControl [MEMORY] hold= 11,423,744 used= 11,358,208 count= 4 avg_used= 2,839,552 block_cnt= 4 chunk_cnt= 4 mod=DiskIteratorSto [MEMORY] hold= 8,895,632 used= 8,810,249 count= 79 avg_used= 111,522 block_cnt= 25 chunk_cnt= 5 mod=OmtTenant [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=RCSrv [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=ArcFetchQueue [MEMORY] hold= 8,486,912 used= 8,426,528 count= 37 avg_used= 227,744 block_cnt= 37 chunk_cnt= 36 mod=SQLSessionInfo [MEMORY] hold= 8,257,536 used= 8,240,064 count= 2 avg_used= 4,120,032 block_cnt= 2 chunk_cnt= 2 mod=FreeTbltStream [MEMORY] hold= 8,151,040 used= 8,131,072 count= 1 avg_used= 8,131,072 block_cnt= 1 chunk_cnt= 1 mod=TransService [MEMORY] hold= 7,028,736 used= 6,998,656 count= 107 avg_used= 65,408 block_cnt= 107 chunk_cnt= 29 mod=MemTblObj [MEMORY] hold= 5,980,160 used= 3,118,560 count= 365 avg_used= 8,544 block_cnt= 365 chunk_cnt= 212 mod=[T]ObDSActionAr [MEMORY] hold= 5,644,288 used= 5,625,088 count= 86 avg_used= 65,408 block_cnt= 86 chunk_cnt= 55 mod=DDLKvMgrObj [MEMORY] hold= 4,739,072 used= 4,718,712 count= 1 avg_used= 4,718,712 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMonMap [MEMORY] hold= 4,137,600 used= 4,080,336 count= 507 avg_used= 8,048 block_cnt= 507 chunk_cnt= 38 mod=[T]ObPerfEventR [MEMORY] hold= 4,096,000 used= 4,063,232 count= 4 avg_used= 1,015,808 block_cnt= 4 chunk_cnt= 4 mod=CommitVersions [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMemCU [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 12 mod=CkptDgnMem [MEMORY] hold= 3,145,728 used= 2,228,224 count= 128 avg_used= 17,408 block_cnt= 128 chunk_cnt= 5 mod=SqlDtlQueue [MEMORY] hold= 2,883,584 used= 2,879,744 count= 2 avg_used= 1,439,872 block_cnt= 2 chunk_cnt= 2 mod=SqlFltSpanRec [MEMORY] hold= 2,801,920 used= 2,762,752 count= 608 avg_used= 4,544 block_cnt= 608 chunk_cnt= 82 mod=[T]ObTraceEvent [MEMORY] hold= 2,756,608 used= 2,720,005 count= 3 avg_used= 906,668 block_cnt= 3 chunk_cnt= 2 mod=SqlDtl1stBuf [MEMORY] hold= 2,621,440 used= 2,616,320 count= 40 avg_used= 65,408 block_cnt= 40 chunk_cnt= 33 mod=MemTblMgrObj [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=TransTimeWheel [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=XATimeWheel [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 2,490,368 used= 2,468,576 count= 34 avg_used= 72,605 block_cnt= 34 chunk_cnt= 33 mod=bf_queue [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=MediumTabletMap [MEMORY] hold= 2,375,680 used= 2,359,536 count= 2 avg_used= 1,179,768 block_cnt= 2 chunk_cnt= 2 mod=HashBuckLCSta [MEMORY] hold= 2,248,704 used= 2,228,224 count= 1 avg_used= 2,228,224 block_cnt= 1 chunk_cnt= 1 mod=LogIOCb [MEMORY] hold= 2,097,152 used= 2,093,056 count= 32 avg_used= 65,408 block_cnt= 32 chunk_cnt= 32 mod=StorMetaCacheIO [MEMORY] hold= 2,080,768 used= 2,079,744 count= 1 avg_used= 2,079,744 block_cnt= 1 chunk_cnt= 1 mod=LogReplayTask [MEMORY] hold= 2,064,384 used= 2,062,336 count= 2 avg_used= 1,031,168 block_cnt= 2 chunk_cnt= 2 mod=Compressor [MEMORY] hold= 1,925,120 used= 1,885,600 count= 5 avg_used= 377,120 block_cnt= 5 chunk_cnt= 5 mod=ClogGe [MEMORY] hold= 1,777,664 used= 1,770,696 count= 1 avg_used= 1,770,696 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 1,670,592 used= 1,663,936 count= 7 avg_used= 237,705 block_cnt= 7 chunk_cnt= 2 mod=PoolFreeList [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=T3MBucket [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 1,376,256 used= 1,343,488 count= 4 avg_used= 335,872 block_cnt= 4 chunk_cnt= 4 mod=IOWorkerLQ [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPsCache [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPlanCac [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=RewriteRuleMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPsInfo [MEMORY] hold= 933,888 used= 931,072 count= 1 avg_used= 931,072 block_cnt= 1 chunk_cnt= 1 mod=ArcSenderQueue [MEMORY] hold= 917,504 used= 915,712 count= 14 avg_used= 65,408 block_cnt= 14 chunk_cnt= 14 mod=TmpBlockManager [MEMORY] hold= 829,632 used= 804,864 count= 258 avg_used= 3,119 block_cnt= 130 chunk_cnt= 2 mod=LSMap [MEMORY] hold= 802,944 used= 786,432 count= 256 avg_used= 3,072 block_cnt= 128 chunk_cnt= 3 mod=TabletToLS [MEMORY] hold= 786,752 used= 524,352 count= 36 avg_used= 14,565 block_cnt= 34 chunk_cnt= 5 mod=LogAlloc [MEMORY] hold= 699,184 used= 693,488 count= 89 avg_used= 7,792 block_cnt= 89 chunk_cnt= 25 mod=CkptDgnMemNode [MEMORY] hold= 692,768 used= 687,104 count= 88 avg_used= 7,808 block_cnt= 88 chunk_cnt= 73 mod=HashNodeLCSta [MEMORY] hold= 663,552 used= 655,360 count= 1 avg_used= 655,360 block_cnt= 1 chunk_cnt= 1 mod=FetchLog [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DupTbLease [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTF [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTR [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DetectorTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MultiVersionGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MdsT [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=ElectTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=OBJLockGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=FrzTrigger [MEMORY] hold= 614,400 used= 553,040 count= 10 avg_used= 55,304 block_cnt= 10 chunk_cnt= 5 mod=TabletIDSetBkt [MEMORY] hold= 608,960 used= 599,168 count= 151 avg_used= 3,968 block_cnt= 80 chunk_cnt= 30 mod=CkpDgn [MEMORY] hold= 606,208 used= 589,872 count= 2 avg_used= 294,936 block_cnt= 2 chunk_cnt= 1 mod=TmpBlockMap [MEMORY] hold= 598,016 used= 590,232 count= 1 avg_used= 590,232 block_cnt= 1 chunk_cnt= 1 mod=DagNetIdMap [MEMORY] hold= 589,824 used= 569,088 count= 6 avg_used= 94,848 block_cnt= 6 chunk_cnt= 6 mod=ResourceGroup [MEMORY] hold= 511,936 used= 493,990 count= 14 avg_used= 35,285 block_cnt= 14 chunk_cnt= 12 mod=SqlMemMgr [MEMORY] hold= 450,560 used= 422,240 count= 5 avg_used= 84,448 block_cnt= 5 chunk_cnt= 4 mod=LSSvr [MEMORY] hold= 450,560 used= 403,744 count= 11 avg_used= 36,704 block_cnt= 11 chunk_cnt= 11 mod=CommSysVarFac [MEMORY] hold= 447,312 used= 407,904 count= 607 avg_used= 672 block_cnt= 273 chunk_cnt= 64 mod=[T]ObWarningBuf [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ApplySrv [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ReplaySrv [MEMORY] hold= 370,192 used= 366,976 count= 47 avg_used= 7,808 block_cnt= 47 chunk_cnt= 37 mod=TabletIDSetNode [MEMORY] hold= 314,880 used= 312,320 count= 40 avg_used= 7,808 block_cnt= 40 chunk_cnt= 36 mod=HashNodePlanCac [MEMORY] hold= 310,480 used= 166,424 count= 1,980 avg_used= 84 block_cnt= 162 chunk_cnt= 48 mod=Coro [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=ColUsagHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=TmpMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=DmlStatHashMap [MEMORY] hold= 291,344 used= 285,064 count= 36 avg_used= 7,918 block_cnt= 36 chunk_cnt= 14 mod=TabletStats [MEMORY] hold= 266,112 used= 261,888 count= 66 avg_used= 3,968 block_cnt= 45 chunk_cnt= 30 mod=TinyAllocator [MEMORY] hold= 264,000 used= 261,888 count= 33 avg_used= 7,936 block_cnt= 33 chunk_cnt= 30 mod=SqlSession [MEMORY] hold= 262,144 used= 261,632 count= 4 avg_used= 65,408 block_cnt= 4 chunk_cnt= 4 mod=TmpPageCache [MEMORY] hold= 262,144 used= 261,632 count= 4 avg_used= 65,408 block_cnt= 4 chunk_cnt= 4 mod=LockMemObj [MEMORY] hold= 256,016 used= 253,952 count= 32 avg_used= 7,936 block_cnt= 32 chunk_cnt= 32 mod=LSIter [MEMORY] hold= 249,408 used= 211,752 count= 519 avg_used= 408 block_cnt= 516 chunk_cnt= 90 mod=[T]ObTLDecoderC [MEMORY] hold= 246,160 used= 212,992 count= 512 avg_used= 416 block_cnt= 68 chunk_cnt= 13 mod=OMT_Worker [MEMORY] hold= 245,760 used= 243,840 count= 5 avg_used= 48,768 block_cnt= 5 chunk_cnt= 4 mod=palf_env [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagNetMap [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagMap [MEMORY] hold= 200,832 used= 196,608 count= 64 avg_used= 3,072 block_cnt= 32 chunk_cnt= 4 mod=TxCtxMgr [MEMORY] hold= 147,456 used= 139,264 count= 1 avg_used= 139,264 block_cnt= 1 chunk_cnt= 1 mod=RFLTaskQueue [MEMORY] hold= 147,456 used= 131,072 count= 2 avg_used= 65,536 block_cnt= 2 chunk_cnt= 2 mod=SHADiagMgr [MEMORY] hold= 147,456 used= 131,072 count= 2 avg_used= 65,536 block_cnt= 2 chunk_cnt= 2 mod=SuspectInfoMgr [MEMORY] hold= 128,880 used= 97,740 count= 11 avg_used= 8,885 block_cnt= 11 chunk_cnt= 11 mod=SqlSessiQuerSql [MEMORY] hold= 126,048 used= 31,200 count= 1,300 avg_used= 24 block_cnt= 192 chunk_cnt= 51 mod=[T]MemoryContex [MEMORY] hold= 114,688 used= 106,496 count= 1 avg_used= 106,496 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlMgr [MEMORY] hold= 109,088 used= 108,192 count= 14 avg_used= 7,728 block_cnt= 14 chunk_cnt= 12 mod=HashNodePsCache [MEMORY] hold= 98,304 used= 94,976 count= 1 avg_used= 94,976 block_cnt= 1 chunk_cnt= 1 mod=PxPool [MEMORY] hold= 90,112 used= 82,048 count= 1 avg_used= 82,048 block_cnt= 1 chunk_cnt= 1 mod=MetaMemMgr [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 11 mod=PlanVaIdx [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 11 mod=SqlSessiVarMap [MEMORY] hold= 81,920 used= 57,920 count= 5 avg_used= 11,584 block_cnt= 5 chunk_cnt= 4 mod=LogApplyStatus [MEMORY] hold= 81,920 used= 68,480 count= 5 avg_used= 13,696 block_cnt= 5 chunk_cnt= 4 mod=LogReplayStatus [MEMORY] hold= 81,920 used= 51,240 count= 5 avg_used= 10,248 block_cnt= 5 chunk_cnt= 5 mod=TabletSvrBucket [MEMORY] hold= 80,000 used= 79,360 count= 10 avg_used= 7,936 block_cnt= 10 chunk_cnt= 8 mod=LSLocCache [MEMORY] hold= 78,720 used= 78,080 count= 10 avg_used= 7,808 block_cnt= 10 chunk_cnt= 9 mod=HashNodePsInfo [MEMORY] hold= 73,728 used= 65,536 count= 1 avg_used= 65,536 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisMgr [MEMORY] hold= 73,728 used= 66,176 count= 1 avg_used= 66,176 block_cnt= 1 chunk_cnt= 1 mod=Rpc [MEMORY] hold= 73,728 used= 72,736 count= 1 avg_used= 72,736 block_cnt= 1 chunk_cnt= 1 mod=LogSharedQueueT [MEMORY] hold= 73,536 used= 73,344 count= 2 avg_used= 36,672 block_cnt= 2 chunk_cnt= 2 mod=FlushMeta [MEMORY] hold= 65,984 used= 56,896 count= 140 avg_used= 406 block_cnt= 130 chunk_cnt= 47 mod=NegParamIdex [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxDataMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxCtxMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=FlushLog [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=DDLKVObj [MEMORY] hold= 64,000 used= 63,488 count= 8 avg_used= 7,936 block_cnt= 8 chunk_cnt= 5 mod=LCLSender [MEMORY] hold= 63,744 used= 63,232 count= 8 avg_used= 7,904 block_cnt= 8 chunk_cnt= 8 mod=SqlSessionSbloc [MEMORY] hold= 62,848 used= 62,336 count= 8 avg_used= 7,792 block_cnt= 8 chunk_cnt= 7 mod=CkptDgnCUNode [MEMORY] hold= 56,000 used= 55,552 count= 7 avg_used= 7,936 block_cnt= 7 chunk_cnt= 6 mod=NotParamInfo [MEMORY] hold= 48,832 used= 44,840 count= 2 avg_used= 22,420 block_cnt= 2 chunk_cnt= 2 mod=DynamicFactor [MEMORY] hold= 42,240 used= 41,472 count= 12 avg_used= 3,456 block_cnt= 12 chunk_cnt= 7 mod=CACHE_INST [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=Autoincrement [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 4 mod=MatchLsnMap [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 5 mod=MdsTableMgr [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 2 mod=ObDMMDL [MEMORY] hold= 32,768 used= 24,688 count= 2 avg_used= 12,344 block_cnt= 2 chunk_cnt= 1 mod=TLD_ClientTask [MEMORY] hold= 32,768 used= 17,408 count= 2 avg_used= 8,704 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTTP [MEMORY] hold= 32,768 used= 25,600 count= 1 avg_used= 25,600 block_cnt= 1 chunk_cnt= 1 mod=TSQLSessionMgr [MEMORY] hold= 32,000 used= 31,744 count= 4 avg_used= 7,936 block_cnt= 4 chunk_cnt= 4 mod=ShareBlksMap [MEMORY] hold= 27,200 used= 25,600 count= 25 avg_used= 1,024 block_cnt= 17 chunk_cnt= 13 mod=TGTimer [MEMORY] hold= 24,864 used= 572 count= 286 avg_used= 2 block_cnt= 224 chunk_cnt= 158 mod=[T]TSIUseWeak [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 23,616 used= 23,424 count= 3 avg_used= 7,808 block_cnt= 3 chunk_cnt= 2 mod=DirToBlk [MEMORY] hold= 23,520 used= 23,160 count= 5 avg_used= 4,632 block_cnt= 5 chunk_cnt= 4 mod=TTLTaskBucket [MEMORY] hold= 16,768 used= 8,448 count= 3 avg_used= 2,816 block_cnt= 2 chunk_cnt= 2 mod=BaseLogWriter [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TblCtxIMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TableCtxMgr [MEMORY] hold= 16,384 used= 10,248 count= 1 avg_used= 10,248 block_cnt= 1 chunk_cnt= 1 mod=TmpMemBlkMgr [MEMORY] hold= 16,384 used= 8,448 count= 1 avg_used= 8,448 block_cnt= 1 chunk_cnt= 1 mod=PalfEnv [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_AssignedMgr [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=backupTaskSched [MEMORY] hold= 15,744 used= 15,616 count= 2 avg_used= 7,808 block_cnt= 2 chunk_cnt= 2 mod=SuspectInfoNode [MEMORY] hold= 15,680 used= 15,360 count= 5 avg_used= 3,072 block_cnt= 5 chunk_cnt= 5 mod=LockMemtableMgr [MEMORY] hold= 12,576 used= 12,440 count= 2 avg_used= 6,220 block_cnt= 2 chunk_cnt= 2 mod=PxPoolBkt [MEMORY] hold= 10,368 used= 10,240 count= 2 avg_used= 5,120 block_cnt= 2 chunk_cnt= 2 mod=PoolArenaArray [MEMORY] hold= 10,240 used= 10,112 count= 2 avg_used= 5,056 block_cnt= 1 chunk_cnt= 1 mod=HAScheduler [MEMORY] hold= 10,240 used= 10,112 count= 2 avg_used= 5,056 block_cnt= 1 chunk_cnt= 1 mod=Scheduler [MEMORY] hold= 9,296 used= 9,160 count= 2 avg_used= 4,580 block_cnt= 2 chunk_cnt= 2 mod=StoLocCache [MEMORY] hold= 8,080 used= 7,720 count= 5 avg_used= 1,544 block_cnt= 5 chunk_cnt= 4 mod=TXBFRetryId [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MdsDebugMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=APPLY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=REPLAY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LockWaitMgr [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ls_id [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LsElectRefInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DASIDCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=PurgeThrottle [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=FetchLogTask [MEMORY] hold= 7,856 used= 7,792 count= 1 avg_used= 7,792 block_cnt= 1 chunk_cnt= 1 mod=ColUsagNode [MEMORY] hold= 7,776 used= 7,712 count= 1 avg_used= 7,712 block_cnt= 1 chunk_cnt= 1 mod=DmlStatNode [MEMORY] hold= 7,488 used= 7,424 count= 1 avg_used= 7,424 block_cnt= 1 chunk_cnt= 1 mod=LogIOWS [MEMORY] hold= 6,416 used= 6,216 count= 3 avg_used= 2,072 block_cnt= 2 chunk_cnt= 2 mod=DetectManager [MEMORY] hold= 5,936 used= 4,032 count= 28 avg_used= 144 block_cnt= 25 chunk_cnt= 16 mod=ObTxLSLogCb [MEMORY] hold= 5,632 used= 4,664 count= 11 avg_used= 424 block_cnt= 10 chunk_cnt= 10 mod=SequenceMap [MEMORY] hold= 5,552 used= 4,664 count= 11 avg_used= 424 block_cnt= 11 chunk_cnt= 10 mod=SequenceIdMap [MEMORY] hold= 5,504 used= 4,664 count= 11 avg_used= 424 block_cnt= 10 chunk_cnt= 9 mod=PackStateMap [MEMORY] hold= 5,456 used= 4,664 count= 11 avg_used= 424 block_cnt= 10 chunk_cnt= 10 mod=ContextsMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResGrpIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResRuleIdMap [MEMORY] hold= 4,240 used= 3,880 count= 5 avg_used= 776 block_cnt= 5 chunk_cnt= 5 mod=DepTabletSet [MEMORY] hold= 4,160 used= 4,096 count= 1 avg_used= 4,096 block_cnt= 1 chunk_cnt= 1 mod=MacroFile [MEMORY] hold= 3,776 used= 3,712 count= 1 avg_used= 3,712 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlDfc [MEMORY] hold= 2,720 used= 2,360 count= 5 avg_used= 472 block_cnt= 4 chunk_cnt= 3 mod=Election [MEMORY] hold= 2,688 used= 2,304 count= 6 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=TableLockNode [MEMORY] hold= 2,352 used= 1,792 count= 8 avg_used= 224 block_cnt= 2 chunk_cnt= 2 mod=LogIOTask [MEMORY] hold= 2,320 used= 1,664 count= 8 avg_used= 208 block_cnt= 8 chunk_cnt= 8 mod=[T]ObMacroBlock [MEMORY] hold= 1,872 used= 1,800 count= 1 avg_used= 1,800 block_cnt= 1 chunk_cnt= 1 mod=PxResMgr [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=MSTXCTX [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TabStatMgrLock [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=DUP_LS_SET [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GCMemtableMap [MEMORY] hold= 1,280 used= 1,088 count= 3 avg_used= 362 block_cnt= 3 chunk_cnt= 3 mod=MustBePosiIdx [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 1,120 used= 1,056 count= 1 avg_used= 1,056 block_cnt= 1 chunk_cnt= 1 mod=LOG_HASH_MAP [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=FlushTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TabShellTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TLD_TIMER [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=DiskUsageTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=CheckPointTimer [MEMORY] hold= 1,056 used= 320 count= 10 avg_used= 32 block_cnt= 9 chunk_cnt= 8 mod=PalfFSCbNode [MEMORY] hold= 944 used= 520 count= 5 avg_used= 104 block_cnt= 5 chunk_cnt= 5 mod=RedoLogBuckLock [MEMORY] hold= 720 used= 648 count= 1 avg_used= 648 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfo [MEMORY] hold= 624 used= 552 count= 1 avg_used= 552 block_cnt= 1 chunk_cnt= 1 mod=LSFreeze [MEMORY] hold= 576 used= 205 count= 5 avg_used= 41 block_cnt= 5 chunk_cnt= 4 mod=KeepAliveBuf [MEMORY] hold= 496 used= 160 count= 5 avg_used= 32 block_cnt= 5 chunk_cnt= 5 mod=PalfRCCbNode [MEMORY] hold= 496 used= 160 count= 5 avg_used= 32 block_cnt= 5 chunk_cnt= 4 mod=RebuildCbNode [MEMORY] hold= 480 used= 120 count= 5 avg_used= 24 block_cnt= 1 chunk_cnt= 1 mod=FreezeTask [MEMORY] hold= 368 used= 186 count= 2 avg_used= 93 block_cnt= 2 chunk_cnt= 2 mod=TxLSLogBuf [MEMORY] hold= 336 used= 144 count= 2 avg_used= 72 block_cnt= 2 chunk_cnt= 1 mod=TntSrvObjPool [MEMORY] hold= 176 used= 112 count= 1 avg_used= 112 block_cnt= 1 chunk_cnt= 1 mod=Coordinator [MEMORY] hold= 160 used= 16 count= 2 avg_used= 8 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTHandler [MEMORY] hold= 128 used= 56 count= 1 avg_used= 56 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfoMap [MEMORY] hold= 80 used= 16 count= 1 avg_used= 16 block_cnt= 1 chunk_cnt= 1 mod=logservice [MEMORY] hold= 18,442,559,472 used= 18,367,596,231 count= 175,710 avg_used= 104,533 mod=SUMMARY [2024-06-25 17:00:18.760868] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=29] [MEMORY] tenant_id= 1004 ctx_id= MEMSTORE_CTX_ID hold= 55,475,961,856 used= 55,042,555,904 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 55,042,555,904 used= 55,017,372,648 count= 26,453 avg_used= 2,079,816 block_cnt= 26,453 chunk_cnt= 26,453 mod=Memstore [MEMORY] hold= 55,042,555,904 used= 55,017,372,648 count= 26,453 avg_used= 2,079,816 mod=SUMMARY [2024-06-25 17:00:18.760876] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1004 ctx_id= TRANS_CTX_MGR_ID hold= 6,291,456 used= 3,645,440 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 3,645,440 used= 3,612,480 count= 5 avg_used= 722,496 block_cnt= 5 chunk_cnt= 3 mod=PartTranCtxMgr [MEMORY] hold= 3,645,440 used= 3,612,480 count= 5 avg_used= 722,496 mod=SUMMARY [2024-06-25 17:00:18.760885] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1004 ctx_id= PLAN_CACHE_CTX_ID hold= 1,232,424,960 used= 1,205,544,208 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 1,090,557,456 used= 1,016,422,555 count= 46,115 avg_used= 22,041 block_cnt= 46,115 chunk_cnt= 566 mod=SqlPhyPlan [MEMORY] hold= 66,337,664 used= 61,810,624 count= 3,844 avg_used= 16,079 block_cnt= 3,844 chunk_cnt= 545 mod=SqlPlanCache [MEMORY] hold= 28,889,920 used= 28,656,896 count= 3,611 avg_used= 7,936 block_cnt= 3,611 chunk_cnt= 207 mod=OB_SQLSTAT_OBJ [MEMORY] hold= 18,520,320 used= 17,937,344 count= 9,041 avg_used= 1,984 block_cnt= 2,270 chunk_cnt= 510 mod=CreateContext [MEMORY] hold= 1,237,120 used= 1,196,088 count= 140 avg_used= 8,543 block_cnt= 140 chunk_cnt= 2 mod=SqlPhyPlObj [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=PlanCache [MEMORY] hold= 1,205,544,208 used= 1,126,025,107 count= 62,753 avg_used= 17,943 mod=SUMMARY [2024-06-25 17:00:18.760896] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1004 ctx_id= GLIBC hold= 4,194,304 used= 2,327,824 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 2,278,992 used= 1,819,125 count= 6,525 avg_used= 278 block_cnt= 290 chunk_cnt= 2 mod=PlJit [MEMORY] hold= 48,832 used= 45,810 count= 42 avg_used= 1,090 block_cnt= 13 chunk_cnt= 1 mod=PlCodeGen [MEMORY] hold= 2,327,824 used= 1,864,935 count= 6,567 avg_used= 283 mod=SUMMARY [2024-06-25 17:00:18.760904] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1004 ctx_id= CO_STACK hold= 683,671,552 used= 670,924,800 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 670,924,800 used= 669,676,800 count= 1,300 avg_used= 515,136 block_cnt= 1,300 chunk_cnt= 326 mod=CoStack [MEMORY] hold= 670,924,800 used= 669,676,800 count= 1,300 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.760913] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1004 ctx_id= KVSTORE_CACHE_ID hold= 541,409,148,928 used= 541,409,148,928 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 541,409,148,928 used= 0 count= 258,164 avg_used= 0 block_cnt= 0 chunk_cnt= 0 mod=KvstorCacheMb [MEMORY] hold= 541,409,148,928 used= 0 count= 258,164 avg_used= 0 mod=SUMMARY [2024-06-25 17:00:18.760922] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1004 ctx_id= META_OBJ_CTX_ID hold= 165,675,008 used= 162,308,096 limit= 144,955,146,240 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 147,456,000 used= 131,182,000 count= 2,000 avg_used= 65,591 block_cnt= 2,000 chunk_cnt= 74 mod=L_TabletPool [MEMORY] hold= 14,450,688 used= 14,389,760 count= 220 avg_used= 65,408 block_cnt= 220 chunk_cnt= 11 mod=N_TabletPool [MEMORY] hold= 401,408 used= 400,064 count= 2 avg_used= 200,032 block_cnt= 2 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 162,308,096 used= 145,971,824 count= 2,222 avg_used= 65,693 mod=SUMMARY [2024-06-25 17:00:18.760930] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1004 ctx_id= TX_CALLBACK_CTX_ID hold= 322,961,408 used= 293,306,816 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 293,306,816 used= 290,941,696 count= 36,661 avg_used= 7,936 block_cnt= 36,661 chunk_cnt= 154 mod=MemtableCallbac [MEMORY] hold= 293,306,816 used= 290,941,696 count= 36,661 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.760939] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1004 ctx_id= LOB_CTX_ID hold= 2,097,152 used= 73,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LobAllocator [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LobPersist [MEMORY] hold= 73,536 used= 73,344 count= 2 avg_used= 36,672 mod=SUMMARY [2024-06-25 17:00:18.760947] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1004 ctx_id= PS_CACHE_CTX_ID hold= 6,291,456 used= 3,809,376 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 3,809,376 used= 3,267,545 count= 1,998 avg_used= 1,635 block_cnt= 351 chunk_cnt= 3 mod=SqlPsCache [MEMORY] hold= 3,809,376 used= 3,267,545 count= 1,998 avg_used= 1,635 mod=SUMMARY [2024-06-25 17:00:18.760958] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1004 ctx_id= TX_DATA_TABLE hold= 2,053,193,728 used= 1,985,541,824 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 1,953,873,472 used= 1,938,121,984 count= 244,219 avg_used= 7,936 block_cnt= 244,219 chunk_cnt= 967 mod=TX_DATA_SLICE [MEMORY] hold= 31,628,352 used= 31,497,280 count= 15 avg_used= 2,099,818 block_cnt= 15 chunk_cnt= 12 mod=MEMTABLE_ARENA [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 2 mod=TX_DATA_ARENA [MEMORY] hold= 1,985,541,824 used= 1,969,658,944 count= 244,239 avg_used= 8,064 mod=SUMMARY [2024-06-25 17:00:18.760968] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1004 ctx_id= MDS_DATA_ID hold= 20,971,520 used= 256,000 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 256,000 used= 253,952 count= 32 avg_used= 7,936 block_cnt= 32 chunk_cnt= 10 mod=MdsTable [MEMORY] hold= 256,000 used= 253,952 count= 32 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.761008] INFO [STORAGE] print_tenant_usage (ob_tenant_memory_printer.cpp:125) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=4] ====== tenants memory info ====== === TENANTS MEMORY INFO === all_tenants_memstore_used= 55,765,368,832, divisive_memory_used= 462,983,168 [TENANT_MEMORY] tenant_id= 500 mem_tenant_limit= 9,223,372,036,854,775,807 mem_tenant_hold= 3,474,333,696 kv_cache_mem= 0 [TENANT_MEMORY] tenant_id= 508 mem_tenant_limit= 1,073,741,824 mem_tenant_hold= 40,296,448 kv_cache_mem= 0 [TENANT_MEMORY] tenant_id= 1 now= 1,719,306,018,758,995 active_memstore_used= 285,212,672 total_memstore_used= 285,212,672 total_memstore_hold= 285,212,672 memstore_freeze_trigger_limit= 4,039,953,600 memstore_limit= 20,199,768,050 mem_tenant_limit= 40,399,536,128 mem_tenant_hold= 17,737,887,744 kv_cache_mem= 14,854,127,616 max_mem_memstore_can_get_now= 0 memstore_alloc_pos= 19,216,203,776 memstore_frozen_pos= 18,930,991,104 memstore_reclaimed_pos= 18,930,991,104 [TENANT_MEMORY] tenant_id= 1,001 now= 1,719,306,018,758,995 active_memstore_used= 0 total_memstore_used= 0 total_memstore_hold= 0 memstore_freeze_trigger_limit= 107,374,180 memstore_limit= 536,870,900 mem_tenant_limit= 1,073,741,824 mem_tenant_hold= 343,166,976 kv_cache_mem= 14,680,064 max_mem_memstore_can_get_now= 0 memstore_alloc_pos= 0 memstore_frozen_pos= 0 memstore_reclaimed_pos= 0 [TENANT_MEMORY] tenant_id= 1,002 now= 1,719,306,018,758,995 active_memstore_used= 4,194,304 total_memstore_used= 4,194,304 total_memstore_hold= 4,194,304 memstore_freeze_trigger_limit= 53,542,900 memstore_limit= 536,870,900 mem_tenant_limit= 1,073,741,824 mem_tenant_hold= 933,953,536 kv_cache_mem= 123,731,968 max_mem_memstore_can_get_now= 0 memstore_alloc_pos= 308,054,851,584 memstore_frozen_pos= 308,050,657,280 memstore_reclaimed_pos= 308,050,657,280 [TENANT_MEMORY] tenant_id= 1,003 now= 1,719,306,018,760,043 active_memstore_used= 0 total_memstore_used= 0 total_memstore_hold= 0 memstore_freeze_trigger_limit= 8,053,063,680 memstore_limit= 40,265,318,400 mem_tenant_limit= 80,530,636,800 mem_tenant_hold= 845,557,760 kv_cache_mem= 12,582,912 max_mem_memstore_can_get_now= 0 memstore_alloc_pos= 0 memstore_frozen_pos= 0 memstore_reclaimed_pos= 0 [TENANT_MEMORY] tenant_id= 1,004 now= 1,719,306,018,760,043 active_memstore_used= 55,475,961,856 total_memstore_used= 55,475,961,856 total_memstore_hold= 55,475,961,856 memstore_freeze_trigger_limit= 72,477,573,120 memstore_limit= 362,387,865,600 mem_tenant_limit= 724,775,731,200 mem_tenant_hold= 622,220,120,064 kv_cache_mem= 541,409,148,928 max_mem_memstore_can_get_now= 0 memstore_alloc_pos= 58,193,176,690,688 memstore_frozen_pos= 58,137,700,728,832 memstore_reclaimed_pos= 58,137,700,728,832 [2024-06-25 17:00:18.761071] INFO [STORAGE] print_tenant_usage (ob_tenant_memory_printer.cpp:172) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=7] [CHUNK_MGR] free=182 pushes=10235828566 pops=10235828384 limit=939,524,096,000 hold=646,107,275,264 total_hold=646,765,871,104 used=645,595,316,224 freelist_hold= 511,959,040 large_freelist_hold= 130,277,376 maps= 5,113,416 unmaps= 4,806,942 large_maps= 2,211 large_unmaps= 2,011 huge_maps= 4,753,214 huge_unmaps= 4,753,121 memalign=0 resident_size=647,414,452,224 virtual_memory_used=647,946,121,216 [2024-06-25 17:00:18.761131] INFO print (ob_malloc_time_monitor.cpp:57) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=8] [MALLOC_TIME_MONITOR] show the distribution of ob_malloc's cost_time [MALLOC_TIME_MONITOR] [ 0, 10): delta_total_cost_time= 2621021, delta_count= 33571495, avg_cost_time= 0 [MALLOC_TIME_MONITOR] [ 10, 100): delta_total_cost_time= 945, delta_count= 44, avg_cost_time= 21 [MALLOC_TIME_MONITOR] [ 100, 1000): delta_total_cost_time= 4827, delta_count= 13, avg_cost_time= 371 [MALLOC_TIME_MONITOR] [ 1000, 10000): delta_total_cost_time= 0, delta_count= 0, avg_cost_time= 0 [MALLOC_TIME_MONITOR] [ 10000, 100000): delta_total_cost_time= 0, delta_count= 0, avg_cost_time= 0 [MALLOC_TIME_MONITOR] [ 100000, 1000000): delta_total_cost_time= 0, delta_count= 0, avg_cost_time= 0 [MALLOC_TIME_MONITOR] [ 1000000, 9223372036854775807): delta_total_cost_time= 0, delta_count= 0, avg_cost_time= 0 [2024-06-25 17:00:18.762227] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=4] Dumping All Cache Objs(alloc_obj_list.count()=717, alloc_obj_list=[{obj_id:14, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176366}, {obj_id:16, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:449104}, {obj_id:17, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303715}, {obj_id:18, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:248080}, {obj_id:19, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271604}, {obj_id:20, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216033}, {obj_id:21, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191870}, {obj_id:22, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:23, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179794}, {obj_id:24, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230228}, {obj_id:25, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188271}, {obj_id:26, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196930}, {obj_id:28, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174576}, {obj_id:29, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301933}, {obj_id:33, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:34, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178768}, {obj_id:35, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211414}, {obj_id:36, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224486}, {obj_id:41, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188622}, {obj_id:42, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199959}, {obj_id:43, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199095}, {obj_id:44, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199093}, {obj_id:45, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193076}, {obj_id:46, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214118}, {obj_id:47, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199546}, {obj_id:48, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199766}, {obj_id:49, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190029}, {obj_id:50, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186863}, {obj_id:51, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185594}, {obj_id:52, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200193}, {obj_id:53, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193157}, {obj_id:54, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188399}, {obj_id:55, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185761}, {obj_id:56, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:57, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185793}, {obj_id:58, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:59, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185738}, {obj_id:60, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192013}, {obj_id:61, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187261}, {obj_id:62, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183532}, {obj_id:63, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210345}, {obj_id:64, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183340}, {obj_id:65, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188170}, {obj_id:66, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183730}, {obj_id:67, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191888}, {obj_id:68, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183436}, {obj_id:69, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184586}, {obj_id:70, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188962}, {obj_id:71, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:308297}, {obj_id:72, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:251495}, {obj_id:73, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:304658}, {obj_id:74, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247858}, {obj_id:75, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201160}, {obj_id:77, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:81, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176269}, {obj_id:83, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:84, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:85, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:86, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:87, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:89, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:90, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:91, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:92, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:94, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183659}, {obj_id:97, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:100, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:103, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:106, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196176}, {obj_id:107, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188336}, {obj_id:108, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:275361}, {obj_id:109, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179806}, {obj_id:110, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191447}, {obj_id:111, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:275362}, {obj_id:112, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218570}, {obj_id:116, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:118, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:684299}, {obj_id:119, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:120, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190714}, {obj_id:121, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223442}, {obj_id:136, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177206}, {obj_id:137, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188473}, {obj_id:141, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238685}, {obj_id:148, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178773}, {obj_id:149, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174899}, {obj_id:151, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174973}, {obj_id:152, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174974}, {obj_id:153, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174899}, {obj_id:154, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176141}, {obj_id:155, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179296}, {obj_id:156, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178110}, {obj_id:158, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:159, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:160, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175011}, {obj_id:161, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175011}, {obj_id:162, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175061}, {obj_id:163, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175061}, {obj_id:164, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:165, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:167, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:168, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181212}, {obj_id:169, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177355}, {obj_id:170, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178587}, {obj_id:172, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177556}, {obj_id:173, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178796}, {obj_id:174, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:175, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176287}, {obj_id:176, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176601}, {obj_id:177, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:178, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177809}, {obj_id:179, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178207}, {obj_id:180, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179455}, {obj_id:181, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176359}, {obj_id:182, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177615}, {obj_id:183, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178110}, {obj_id:184, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176293}, {obj_id:185, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182472}, {obj_id:186, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181294}, {obj_id:187, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181212}, {obj_id:188, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178855}, {obj_id:189, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179247}, {obj_id:190, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180179}, {obj_id:191, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174869}, {obj_id:192, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176141}, {obj_id:193, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178955}, {obj_id:194, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179556}, {obj_id:195, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176751}, {obj_id:196, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182472}, {obj_id:197, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176028}, {obj_id:198, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176242}, {obj_id:199, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177698}, {obj_id:200, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177506}, {obj_id:201, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176601}, {obj_id:202, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178207}, {obj_id:203, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176359}, {obj_id:204, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179988}, {obj_id:205, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179191}, {obj_id:206, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:383555}, {obj_id:275, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:291, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176429}, {obj_id:301, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195808}, {obj_id:349, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:435, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:284362}, {obj_id:438, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277099}, {obj_id:441, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:365724}, {obj_id:442, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:352792}, {obj_id:477, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294946}, {obj_id:490, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210456}, {obj_id:501, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:326767}, {obj_id:503, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230400}, {obj_id:504, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185730}, {obj_id:506, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:12123}, {obj_id:507, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1069986}, {obj_id:508, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173197}, {obj_id:509, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199042}, {obj_id:510, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201819}, {obj_id:511, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197302}, {obj_id:513, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231048}, {obj_id:514, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222216}, {obj_id:515, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:516, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:519, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:520, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:340122}, {obj_id:635, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:281722}, {obj_id:636, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:313618}, {obj_id:637, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:404380}, {obj_id:638, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:513636}, {obj_id:642, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:406569}, {obj_id:643, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:526721}, {obj_id:644, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:324772}, {obj_id:645, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:385343}, {obj_id:652, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:336141}, {obj_id:653, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:407607}, {obj_id:656, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:302133}, {obj_id:657, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:349539}, {obj_id:718, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234380}, {obj_id:1036, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286678}, {obj_id:1093, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:316722}, {obj_id:1119, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:306729}, {obj_id:1131, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:226842}, {obj_id:1134, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1135, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204208}, {obj_id:1137, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1170, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1172, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1180, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1181, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1184, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:360170}, {obj_id:1205, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:226578}, {obj_id:1209, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234762}, {obj_id:1210, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175923}, {obj_id:1217, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185026}, {obj_id:1219, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186335}, {obj_id:1223, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186868}, {obj_id:1224, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187165}, {obj_id:1230, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196301}, {obj_id:1231, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:333432}, {obj_id:1242, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1250, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184636}, {obj_id:1277, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219147}, {obj_id:1280, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183351}, {obj_id:1281, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181880}, {obj_id:1282, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180188}, {obj_id:1283, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:343446}, {obj_id:1349, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212498}, {obj_id:1359, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200164}, {obj_id:1368, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:409123}, {obj_id:1371, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:527267}, {obj_id:1394, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212873}, {obj_id:1396, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194252}, {obj_id:1399, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194540}, {obj_id:1420, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:263353}, {obj_id:1421, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276973}, {obj_id:1428, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:349054}, {obj_id:1429, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:411748}, {obj_id:1432, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:439302}, {obj_id:1436, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276882}, {obj_id:1438, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298234}, {obj_id:1439, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:553609}, {obj_id:1441, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:425770}, {obj_id:1443, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:532331}, {obj_id:1444, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:441552}, {obj_id:1445, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:557166}, {obj_id:1446, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235923}, {obj_id:1447, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234251}, {obj_id:1449, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229611}, {obj_id:1471, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:249500}, {obj_id:1472, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303921}, {obj_id:1473, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:258588}, {obj_id:1475, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320137}, {obj_id:1480, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282443}, {obj_id:1481, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:362392}, {obj_id:1488, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286101}, {obj_id:1489, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:368900}, {obj_id:1493, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:274640}, {obj_id:1494, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294693}, {obj_id:1497, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:378392}, {obj_id:1498, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:457842}, {obj_id:1499, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:437052}, {obj_id:1501, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:428020}, {obj_id:1502, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:550068}, {obj_id:1503, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:412238}, {obj_id:1504, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:362578}, {obj_id:1505, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:535888}, {obj_id:1506, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:511046}, {obj_id:1507, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:433015}, {obj_id:1508, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:243979}, {obj_id:1509, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234595}, {obj_id:1510, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232619}, {obj_id:1511, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224335}, {obj_id:1512, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231595}, {obj_id:1513, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223657}, {obj_id:1514, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:346779}, {obj_id:1515, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231905}, {obj_id:1518, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:318041}, {obj_id:1519, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:310697}, {obj_id:1520, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:425065}, {obj_id:1521, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:367809}, {obj_id:1522, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:412013}, {obj_id:1523, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:509887}, {obj_id:1524, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:268389}, {obj_id:1527, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:336988}, {obj_id:1533, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:347500}, {obj_id:1535, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:353092}, {obj_id:1537, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:351196}, {obj_id:1540, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:477250}, {obj_id:1541, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:487084}, {obj_id:1542, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:483796}, {obj_id:1549, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203372}, {obj_id:1555, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:312593}, {obj_id:1556, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:415310}, {obj_id:1559, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207017}, {obj_id:1561, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208699}, {obj_id:1562, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205264}, {obj_id:1564, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216265}, {obj_id:1565, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:215597}, {obj_id:1566, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216631}, {obj_id:1567, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217307}, {obj_id:1569, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205718}, {obj_id:1571, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210530}, {obj_id:1577, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202860}, {obj_id:1578, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221503}, {obj_id:1579, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219106}, {obj_id:1582, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212516}, {obj_id:1586, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184380}, {obj_id:1588, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221044}, {obj_id:1590, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221684}, {obj_id:1591, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320059}, {obj_id:1599, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:1602, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:1619, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213180}, {obj_id:1642, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189353}, {obj_id:1643, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189220}, {obj_id:1644, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193462}, {obj_id:1645, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180107}, {obj_id:1654, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181394}, {obj_id:1657, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181674}, {obj_id:1658, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189842}, {obj_id:1666, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201840}, {obj_id:1668, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181740}, {obj_id:1669, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190036}, {obj_id:1670, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182184}, {obj_id:1708, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200183}, {obj_id:1709, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218850}, {obj_id:1717, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:319852}, {obj_id:1720, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:431890}, {obj_id:1726, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:563003}, {obj_id:1731, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:373531}, {obj_id:1746, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:296698}, {obj_id:1759, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:411595}, {obj_id:1760, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:521508}, {obj_id:1764, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260624}, {obj_id:1765, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:264190}, {obj_id:1766, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176744}, {obj_id:1767, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174432}, {obj_id:1768, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177085}, {obj_id:1769, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181110}, {obj_id:1770, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179110}, {obj_id:1771, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176843}, {obj_id:1773, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198614}, {obj_id:1776, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242568}, {obj_id:1777, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245745}, {obj_id:1778, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:249200}, {obj_id:1779, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:514775}, {obj_id:1780, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174650}, {obj_id:1781, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256438}, {obj_id:1782, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176308}, {obj_id:1783, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:269328}, {obj_id:1784, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241927}, {obj_id:1785, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195211}, {obj_id:1786, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1787, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1789, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:313384}, {obj_id:1790, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189174}, {obj_id:1794, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201926}, {obj_id:1795, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:263611}, {obj_id:1796, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:330941}, {obj_id:1800, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:252092}, {obj_id:1801, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1807, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214817}, {obj_id:1811, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223289}, {obj_id:1823, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320604}, {obj_id:1826, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:268234}, {obj_id:1827, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:369950}, {obj_id:1828, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286091}, {obj_id:1841, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297804}, {obj_id:1842, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:334896}, {obj_id:1848, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212552}, {obj_id:1850, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204050}, {obj_id:1858, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:239469}, {obj_id:1859, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213099}, {obj_id:1860, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:239048}, {obj_id:1868, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229019}, {obj_id:1869, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237507}, {obj_id:1871, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230088}, {obj_id:1872, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238192}, {obj_id:1881, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222260}, {obj_id:1882, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:255313}, {obj_id:1885, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203714}, {obj_id:1886, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205084}, {obj_id:1888, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225301}, {obj_id:1889, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229642}, {obj_id:1892, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237762}, {obj_id:1893, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187591}, {obj_id:1898, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204758}, {obj_id:1899, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213900}, {obj_id:1900, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202713}, {obj_id:1901, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204051}, {obj_id:1903, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204752}, {obj_id:1904, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213464}, {obj_id:1905, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189807}, {obj_id:1906, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189568}, {obj_id:1908, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177216}, {obj_id:1909, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189999}, {obj_id:1910, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190072}, {obj_id:1911, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184744}, {obj_id:1927, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:306481}, {obj_id:1928, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:404932}, {obj_id:1934, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218446}, {obj_id:1935, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:248651}, {obj_id:1939, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245943}, {obj_id:1940, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297508}, {obj_id:1941, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207520}, {obj_id:1942, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206244}, {obj_id:1943, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208189}, {obj_id:1944, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205856}, {obj_id:1945, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209158}, {obj_id:1946, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206074}, {obj_id:1947, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214746}, {obj_id:1948, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206759}, {obj_id:1953, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184677}, {obj_id:1961, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185383}, {obj_id:1962, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:338806}, {obj_id:1963, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:329717}, {obj_id:1964, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:400818}, {obj_id:1965, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:384947}, {obj_id:1966, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:215423}, {obj_id:1967, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193646}, {obj_id:1968, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180032}, {obj_id:1970, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207111}, {obj_id:1971, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207428}, {obj_id:1972, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216076}, {obj_id:1973, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207780}, {obj_id:1982, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191845}, {obj_id:1985, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177576}, {obj_id:1990, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177576}, {obj_id:1994, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186606}, {obj_id:1995, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233374}, {obj_id:1996, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208113}, {obj_id:1998, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231846}, {obj_id:1999, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227382}, {obj_id:2000, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189106}, {obj_id:2001, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197404}, {obj_id:2002, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189506}, {obj_id:2003, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187054}, {obj_id:2004, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188768}, {obj_id:2005, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241398}, {obj_id:2006, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188592}, {obj_id:2007, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185206}, {obj_id:2008, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189703}, {obj_id:2009, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199149}, {obj_id:2010, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276526}, {obj_id:2012, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276526}, {obj_id:2013, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187377}, {obj_id:2015, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189177}, {obj_id:2016, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191033}, {obj_id:2017, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192817}, {obj_id:2018, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203855}, {obj_id:2019, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196646}, {obj_id:2020, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199942}, {obj_id:2021, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203138}, {obj_id:2023, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193422}, {obj_id:2024, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222722}, {obj_id:2025, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194506}, {obj_id:2026, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184181}, {obj_id:2027, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191378}, {obj_id:2028, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206684}, {obj_id:2029, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203947}, {obj_id:2030, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206804}, {obj_id:2031, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204075}, {obj_id:2034, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194701}, {obj_id:2036, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206437}, {obj_id:2038, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196470}, {obj_id:2039, tenant_id:1003, log_del_time:922337 [2024-06-25 17:00:18.762704] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=476] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:18.765391] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.765401] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:18.765407] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.765414] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.765419] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.766200] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:18.766207] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:18.766210] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:18.766212] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:18.766216] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:18.766490] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=3] [APPLY STAT CB APPEND COST TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=32, this=0x2b3a9538a280, extra_info="") [2024-06-25 17:00:18.766499] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=9] [APPLY STAT CB IN QUEUE TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=267, this=0x2b3a9538a4b0, extra_info="") [2024-06-25 17:00:18.766502] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=3] [APPLY STAT CB WAIT COMMIT TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=0, this=0x2b3a9538a6e0, extra_info="") [2024-06-25 17:00:18.766506] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=4] [APPLY STAT CB EXECUTE TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=-521, this=0x2b3a9538a910, extra_info="") [2024-06-25 17:00:18.766511] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=4] [APPLY STAT CB TOTAL TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=-221, this=0x2b3a9538ab40, extra_info="") [2024-06-25 17:00:18.773864] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=9] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289107427328, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:18.775219] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.775228] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.775237] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.775244] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.775248] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.775660] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=1] fetch root ls success(local_cluster_id=1713323324, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.171:2882", sql_port:2881, role:1, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607828518, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.176:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:4, zone:"zone4", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607829360, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.189.192.96:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:5, zone:"zone5", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607830236, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.51:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607831113, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.56:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["10.186.69.171:2882", "10.191.69.51:2882", "10.191.69.56:2882", "10.186.69.176:2882", "10.189.192.96:2882"]) [2024-06-25 17:00:18.776403] WDIAG [RPC] send (ob_poc_rpc_proxy.h:156) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4012] sync rpc execute fail(ret=-4012, addr="10.191.69.52:2882", pcode=1315) [2024-06-25 17:00:18.776416] WDIAG [SQL.ENG] query_statistics (ob_px_tenant_target_monitor.cpp:275) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4012] send rpc to query statistics failed, need send refresh all(ret=-4012) [2024-06-25 17:00:18.776423] WDIAG [SQL.ENG] refresh_statistics (ob_px_tenant_target_monitor.cpp:114) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=6][errcode=-4012] query statistics failed(ret=-4012) [2024-06-25 17:00:18.776427] WDIAG [COMMON] operator() (ob_px_target_mgr.h:158) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] target monitor refresh statistics failed(ret=-4012, px_tenant_info={v:1002}, px_res_info->get_target_monitor()={is_init:true, tenant_id:1002, server:"10.191.69.51:2882", dummy_cache_leader:"10.191.69.52:2882", role:2}) [2024-06-25 17:00:18.777155] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62243][pnio1][T1001][YB420ABF4533-00061661718F652F-0-0] [lt=9][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:18.777780] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=48][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:18.777790] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:18.777795] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=999701, timeout_timestamp=1719306019777146) [2024-06-25 17:00:18.777802] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:18.777804] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:18.777809] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:18.777814] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:18.777817] WDIAG [SQL] open (ob_result_set.cpp:157) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:18.777820] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:18.777825] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.777830] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:18.777836] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=6][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:18.777843] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=6][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.777847] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.777850] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:18.777855] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:18.777860] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4013, ret=-4013) [2024-06-25 17:00:18.777865] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.777870] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:18.777873] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:18.777876] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=3][errcode=-4013] query failed(ret=-4013, conn=0x2b399eda8620, start=1719306018777407, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.777883] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=6][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:18.777886] WDIAG [SHARE.PT] get_by_tenant (ob_persistent_ls_table.cpp:639) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] execute sql failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.777904] WDIAG [SHARE.PT] get_by_tenant (ob_ls_table_operator.cpp:252) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=5][errcode=-4013] get all ls info by persistent_ls_ failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002) [2024-06-25 17:00:18.777909] WDIAG [SHARE.LOCATION] renew_location_for_tenant (ob_ls_location_service.cpp:806) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=5][errcode=-4013] fail to get all ls info(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, ls_infos=[]) [2024-06-25 17:00:18.777914] WDIAG [SHARE.LOCATION] renew_all_ls_locations (ob_ls_location_service.cpp:737) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=5][errcode=-4013] renew cache for tenant failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002) [2024-06-25 17:00:18.781363] WDIAG [SHARE.LOCATION] runTimerTask (ob_location_update_task.cpp:207) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=3][errcode=-4013] fail to renew_all_ls_locations(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.785003] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:18.786907] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.786917] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:18.786922] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.786929] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.786933] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.788679] INFO [STORAGE] update_id_meta (ob_ls_meta.cpp:681) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=10] update id meta(ret=0, service_type=0, limited_id=1719306035353935000, latest_scn={val:1719306018786724001, v:0}, *this={tenant_id:1, ls_id:{id:1}, ls_create_status:1, clog_checkpoint_scn:{val:1719272888853692000, v:0}, clog_base_lsn:{lsn:35095793664}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:1719285609440317000, v:0}, all_id_meta:{id_meta:[{limited_id:1719306035353935000, latest_log_ts:{val:1719306018786724001, v:0}}, {limited_id:80000001, latest_log_ts:{val:1719298896011036001, v:0}}, {limited_id:2660000001, latest_log_ts:{val:1719295281004326001, v:0}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}) [2024-06-25 17:00:18.788700] INFO [STORAGE.TRANS] handle_replay_result (ob_id_service.cpp:238) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=21] handle reply result(ret=0, service_type_=0, last_id=1719306035353935000, limited_id=1719306035353935000, log_ts={val:1719306018786724001, v:0}) [2024-06-25 17:00:18.788706] INFO [STORAGE.TRANS] replay (ob_id_service.cpp:211) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=5] ObIDService replay(service_type_=0, ret=0, ls_log={last_id_:1719306035353935000, limit_id_:1719306035353935000}, base_header={version:1, log_type:7, flag:0, need_pre_replay_barrier:false, need_post_replay_barrier:false, is_compressed:false, replay_hint:0}) [2024-06-25 17:00:18.793070] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.793079] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.793084] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.793091] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.793095] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.796778] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:18.796787] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.796793] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=3][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999958, timeout_timestamp=1719306048796415) [2024-06-25 17:00:18.796800] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:18.796802] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:18.796806] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:18.796810] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:18.796813] WDIAG [SQL] open (ob_result_set.cpp:157) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=2][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:18.796815] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:18.796819] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=3][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT * FROM __all_merge_info WHERE tenant_id = '1001'"}) [2024-06-25 17:00:18.796826] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=5][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT * FROM __all_merge_info WHERE tenant_id = '1001'"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:18.796833] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=7][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:18.796839] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=5][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.796843] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=4][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.796845] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:18.796850] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=3][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_merge_info WHERE tenant_id = '1001'"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:18.796856] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_merge_info WHERE tenant_id = '1001'"}, aret=-4013, ret=-4013) [2024-06-25 17:00:18.796859] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT * FROM __all_merge_info WHERE tenant_id = '1001') [2024-06-25 17:00:18.796863] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:18.796867] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:18.796870] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] query failed(ret=-4013, conn=0x2b3b6150c330, start=1719306018796414, sql=SELECT * FROM __all_merge_info WHERE tenant_id = '1001') [2024-06-25 17:00:18.796875] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:18.796878] WDIAG [SHARE] load_global_merge_info (ob_global_merge_table_operator.cpp:49) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=2][errcode=-4013] fail to execute sql(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1001, meta_tenant_id=1001, sql=SELECT * FROM __all_merge_info WHERE tenant_id = '1001') [2024-06-25 17:00:18.796901] WDIAG [STORAGE] refresh_merge_info (ob_tenant_freeze_info_mgr.cpp:890) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=6][errcode=-4013] failed to load global merge info(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", global_merge_info={tenant_id:1001, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}}) [2024-06-25 17:00:18.796922] WDIAG [STORAGE] runTimerTask (ob_tenant_freeze_info_mgr.cpp:1005) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=20][errcode=-4013] fail to refresh merge info(tmp_ret=-4013, tmp_ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.797586] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:18.797593] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.797598] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=3][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=1999716, timeout_timestamp=1719306020796926) [2024-06-25 17:00:18.797604] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:18.797606] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:18.797609] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=3][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:18.797615] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=5][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:18.797617] WDIAG [SQL] open (ob_result_set.cpp:157) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=2][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:18.797619] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:18.797622] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=3][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}) [2024-06-25 17:00:18.797627] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=5][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:18.797631] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=4][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:18.797638] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=6][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.797640] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=2][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.797645] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=5][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:18.797649] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=2][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:18.797653] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=4][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, aret=-4013, ret=-4013) [2024-06-25 17:00:18.797656] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.797660] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:18.797664] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:18.797667] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=2][errcode=-4013] query failed(ret=-4013, conn=0x2b3a78d44050, start=1719306018797176, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.797671] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:18.797676] WDIAG [SHARE] load (ob_core_table_proxy.cpp:436) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] execute sql failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1001, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.797691] WDIAG [SHARE] load (ob_core_table_proxy.cpp:368) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] load failed(ret=-4013, for_update=false) [2024-06-25 17:00:18.797696] WDIAG [SHARE] get (ob_global_stat_proxy.cpp:442) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] core_table load failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.797699] WDIAG [SHARE] get_snapshot_gc_scn (ob_global_stat_proxy.cpp:164) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] get failed(ret=-4013) [2024-06-25 17:00:18.797703] WDIAG [STORAGE] get_global_info (ob_tenant_freeze_info_mgr.cpp:811) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] fail to get global info(ret=-4013, tenant_id=1001) [2024-06-25 17:00:18.797708] WDIAG [STORAGE] try_update_info (ob_tenant_freeze_info_mgr.cpp:954) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] failed to get global info(ret=-4013) [2024-06-25 17:00:18.797712] WDIAG [STORAGE] runTimerTask (ob_tenant_freeze_info_mgr.cpp:1008) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] fail to try update info(tmp_ret=-4013, tmp_ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.802954] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=51000, remain_us=25924222, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=51, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:18.807546] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.807555] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.807561] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:18.807567] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.807574] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.810305] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95515][T1_L0_G1][T1][YB420ABA45B1-00061645BDA93764-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=12, avg_log_batch_cnt=0, total_group_log_size=4682, avg_group_log_size=390) [2024-06-25 17:00:18.810347] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.810356] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.810360] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:18.810366] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.810371] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.810378] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=14] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=4682) [2024-06-25 17:00:18.811192] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62745][T1_ReplaySrv2][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:18.811294] WDIAG [RPC] send (ob_poc_rpc_proxy.h:156) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=3][errcode=-4012] sync rpc execute fail(ret=-4012, addr="10.191.69.52:2882", pcode=1308) [2024-06-25 17:00:18.811308] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=9][errcode=-4012] rpc task_execute fail(ret=-4012, tenant_id=1002, svr="10.191.69.52:2882", timeout=1999699, timeout_timestamp=1719306018808142) [2024-06-25 17:00:18.811315] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=6][errcode=-4012] task execute failed(ret=-4012) [2024-06-25 17:00:18.811319] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=3][errcode=-4012] execute with sql failed(ret=-4012) [2024-06-25 17:00:18.811324] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=4][errcode=-4012] fail execute plan(ret=-4012) [2024-06-25 17:00:18.811327] WDIAG [SQL] open (ob_result_set.cpp:157) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=2][errcode=-4012] execute plan failed(ret=-4012) [2024-06-25 17:00:18.811330] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=2][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:18.811333] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=2][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}) [2024-06-25 17:00:18.811339] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=5][errcode=-4012] execute failed(ret=-4012, tenant_id=1002, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1718209168713032) [2024-06-25 17:00:18.811344] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=4][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:18.811352] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=6][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:18.811355] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=3][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:18.811358] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=3][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:18.811366] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=4][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:18.811371] INFO [SERVER] process_final (ob_inner_sql_connection.cpp:734) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=5] slow inner sql(last_ret=-4012, sql={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, process_time=2002967) [2024-06-25 17:00:18.811375] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=4][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, aret=-4012, ret=-4012) [2024-06-25 17:00:18.811380] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1002, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.811384] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1002) [2024-06-25 17:00:18.811387] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1002) [2024-06-25 17:00:18.811391] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] query failed(ret=-4012, conn=0x2b3a78d45a30, start=1719306016808398, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.811396] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] read failed(ret=-4012) [2024-06-25 17:00:18.811399] WDIAG [SHARE] load (ob_core_table_proxy.cpp:436) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1002, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.811431] WDIAG [SHARE] load (ob_core_table_proxy.cpp:368) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] load failed(ret=-4012, for_update=false) [2024-06-25 17:00:18.811436] WDIAG [SHARE] get (ob_global_stat_proxy.cpp:442) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] core_table load failed(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:18.811438] WDIAG [SHARE] get_snapshot_gc_scn (ob_global_stat_proxy.cpp:164) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=2][errcode=-4012] get failed(ret=-4012) [2024-06-25 17:00:18.811441] WDIAG [STORAGE] get_global_info (ob_tenant_freeze_info_mgr.cpp:811) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] fail to get global info(ret=-4012, tenant_id=1002) [2024-06-25 17:00:18.811446] WDIAG [STORAGE] try_update_info (ob_tenant_freeze_info_mgr.cpp:954) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] failed to get global info(ret=-4012) [2024-06-25 17:00:18.811448] WDIAG [STORAGE] runTimerTask (ob_tenant_freeze_info_mgr.cpp:1008) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=2][errcode=-4012] fail to try update info(tmp_ret=-4012, tmp_ret="OB_TIMEOUT") [2024-06-25 17:00:18.814142] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.814150] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.814154] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:18.814160] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.814164] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.819924] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.819931] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.819935] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.819940] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.819944] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.821253] INFO do_work (ob_rl_mgr.cpp:709) [62262][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=9] swc wakeup.(stat_period_=1000000, ready=false) [2024-06-25 17:00:18.822262] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=76000, remain_us=23002717, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=76, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:18.822508] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=27] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:18.825065] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABF4533-0006166171EF4863-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301543, send_timestamp=1719305996523521) [2024-06-25 17:00:18.826921] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.826930] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.826935] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.826941] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.826946] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.829583] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:18.829616] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:18.829621] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:18.829623] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:18.829626] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:18.829664] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1039, self="10.191.69.51:2882", committed size=1211) [2024-06-25 17:00:18.829766] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.829774] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.829779] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.829784] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.829790] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.829816] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325962AAE-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305996528272) [2024-06-25 17:00:18.829854] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AAE-0-0] [lt=2] [PALF STAT LOG TASK TIME](palf_id=1039, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=2, avg_log_gen_to_submit_time=28, avg_log_submit_to_flush_time=87, avg_log_submit_to_slide_time=286) [2024-06-25 17:00:18.829887] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AAF-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1005, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=20, avg_log_submit_to_flush_time=92, avg_log_submit_to_slide_time=275) [2024-06-25 17:00:18.832411] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:18.833389] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF59BB-0-0] [lt=4] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1333, avg_group_log_size=121) [2024-06-25 17:00:18.837918] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:18.839508] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.839516] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=8][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.839522] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.839527] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.839530] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.839535] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.839539] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.839545] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.839551] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.839555] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.839557] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=1][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.839562] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.839568] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.839571] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.839574] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.839577] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.839582] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] query failed(ret=-5066, conn=0x2b3a78d45750, start=1719306018838045, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.839587] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.839609] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.839614] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.839620] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.839626] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.839630] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.839635] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.839642] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.839648] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.839652] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.839662] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=0] will sleep(sleep_us=33000, remain_us=360477, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=33, v.err_=-4383, timeout_timestamp=1719306019200138) [2024-06-25 17:00:18.840655] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C7F23FFD2B-0-0] [lt=6] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1005, child={server:"10.186.69.171:2882", region:"sys_region", register_time_us:1719306018839797, keepalive_ts:-1}) [2024-06-25 17:00:18.840661] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C7F23FFD2B-0-0] [lt=6] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1005, self="10.191.69.51:2882", child={server:"10.186.69.171:2882", region:"sys_region", register_time_us:1719306018839797, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:18.840923] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=0][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:18.840935] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=0] will sleep(sleep_us=13000, remain_us=1710975, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=13, v.err_=-4383, timeout_timestamp=1719306020551910) [2024-06-25 17:00:18.844925] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABF4533-0006166172BEBE86-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301554, send_timestamp=1719305996543370) [2024-06-25 17:00:18.847570] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.847578] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.847582] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.847588] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.847593] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.849737] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A07F-0-0] [lt=11] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a5d1db6a0, extra_info="palf_id:1") [2024-06-25 17:00:18.849746] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A07F-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=829, this=0x2b3a5d1db8d0, extra_info="palf_id:1") [2024-06-25 17:00:18.849908] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1333) [2024-06-25 17:00:18.855437] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AB1-0-0] [lt=6] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=89, avg_log_submit_to_slide_time=839) [2024-06-25 17:00:18.855444] INFO [PALF] read (log_cache.cpp:100) [99517][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=3] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:18.856177] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=10093091, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=162, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:18.857297] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5788043, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=190, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:18.859913] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325C300C7-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301543, send_timestamp=1719305996558369) [2024-06-25 17:00:18.860212] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABF4538-00061669548FCBEC-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301547, send_timestamp=1719305996558664) [2024-06-25 17:00:18.860240] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABF4538-00061669548FCBEC-0-0] [lt=9] [PALF STAT LOG TASK TIME](palf_id=1010, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=93, avg_log_submit_to_slide_time=292) [2024-06-25 17:00:18.860247] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1010, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:18.861943] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B2D-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1009, self="10.191.69.51:2882", committed size=1098) [2024-06-25 17:00:18.862021] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1098) [2024-06-25 17:00:18.866259] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:18.866266] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:18.866268] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:18.866271] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:18.866274] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:18.866501] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-000616432547EFEF-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305996564957) [2024-06-25 17:00:18.873288] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.873296] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.873300] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.873305] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.873310] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.874048] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-0006164324713060-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=22301542, send_timestamp=1719305996572505) [2024-06-25 17:00:18.875817] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.875825] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.875829] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.875832] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.875836] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.875840] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.875844] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.875849] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.875855] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.875858] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.875861] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.875867] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.875871] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.875876] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.875879] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.875882] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.875887] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] query failed(ret=-5066, conn=0x2b3a78d44610, start=1719306018874372, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.875892] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.875912] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.875917] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.875922] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.875927] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.875930] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.875933] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.875939] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.875943] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.875949] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.877215] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62236][pnio1][T1001][YB420ABF4533-00061661718F6530-0-0] [lt=9][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:18.877528] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:18.877538] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:18.877544] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=4][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999956, timeout_timestamp=1719306048877079) [2024-06-25 17:00:18.877551] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:18.877553] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:18.877557] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=3][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:18.877561] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:18.877563] WDIAG [SQL] open (ob_result_set.cpp:157) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=2][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:18.879736] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.879743] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.879748] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.879751] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.879754] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.879757] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.879763] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.879768] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.879775] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.879777] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.879779] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.879785] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.879790] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.879793] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.879798] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.879801] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.879804] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] query failed(ret=-5066, conn=0x2b3a78d44330, start=1719306018878005, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.879810] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.879829] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.879834] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.879839] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.879844] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.879848] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.879852] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.879858] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.879863] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.879868] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.885059] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:18.885352] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.885359] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.885364] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.885366] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.885369] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.885373] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.885378] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.885383] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.885390] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.885394] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.885397] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.885402] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.885407] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.885411] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.885415] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.885418] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.885421] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] query failed(ret=-5066, conn=0x2b3afefeaeb0, start=1719306018883929, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.885427] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.885444] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.885449] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.885454] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.885459] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.885462] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.885469] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.885475] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.885479] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.885485] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.888317] INFO pktc_resp_cb_on_msg (pktc_resp.h:70) [62247][pnio1][T0][Y0-0000000000000000-0-0] [lt=8] PNIO resp cb not found: packet_id=1035875065 [2024-06-25 17:00:18.892862] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.894921] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.894928] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.894932] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.894935] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.894938] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.894943] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.894947] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.894953] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.894958] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.894961] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.894964] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.894969] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.894974] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.894978] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.894981] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.894986] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.894989] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] query failed(ret=-5066, conn=0x2b3b6150ceb0, start=1719306018893545, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.894993] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.895012] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.895016] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.895020] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.895025] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.895028] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.895032] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.895038] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.895041] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.895047] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.896067] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100009][T1004_L0_G0][T1004][YB420ABF4534-000616451CB7B7D9-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:18.897616] WDIAG [SERVER] do_heartbeat_event (ob_heartbeat.cpp:265) [62788][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] try reload schema failed(schema_version=1719272886792320, refresh_schema_info={schema_version:1719272886792320, tenant_id:1, sequence_id:1099511641796}, schema_ret=-4023) [2024-06-25 17:00:18.898431] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2710363, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=210, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:18.899397] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-06-25 17:00:18.899406] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=8] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-06-25 17:00:18.899425] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=8] get_max_applied_scn(scn={val:1713323697227896000, v:0}, this={ls_id_:{id:1007}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713323697227896000, v:0}}) [2024-06-25 17:00:18.899435] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=8] get_min_unreplayed_log_info(lsn={lsn:85187186570}, scn={val:1719306018370414005, v:0}, this={ls_id_:{id:1007}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306018850095, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:85187186570}, next_to_submit_scn_:{val:1719306018370414005, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713323697227896000, v:0}, iterator_:{iterator_impl:{buf_:0x2b4185039000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:85187186448}, end_lsn:{lsn:85187186570}, read_buf:{buf_len_:86016, buf_:0x2b4185039000}, block_size:67104768, log_storage_:0x2b3a86d23470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306018370414004, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306018370414004, v:0}, data_checksum:2442502576, flag:1}}, init_mode_version:0, accumulate_checksum:2847000297, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306018370414004, v:0}}}}}) [2024-06-25 17:00:18.900525] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100226][T1004_L0_G0][T1004][YB420ABF4534-000616451CB7B7D9-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:18.900775] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=14834048, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=131, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:18.901097] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=4] re_register_parent reason(palf_id=1009, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306014798883, last_first_register_time_us=1713323716483453, register_time_us=1719306014798883, curr_time_us=1719306018901097) [2024-06-25 17:00:18.901120] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=11] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1009, self="10.191.69.51:2882", reason=1, leader="10.186.69.176:2882", register_time_us=1719306018901109, last_submit_register_req_time_us=1719306018901109, register_parent_reason=1) [2024-06-25 17:00:18.901127] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=7] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1009, self="10.191.69.51:2882") [2024-06-25 17:00:18.903574] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2072317, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=214, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:18.907893] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=52000, remain_us=25819282, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=52, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:18.910246] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [62585][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=13] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-06-25 17:00:18.912479] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.912485] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.912490] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.912492] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.912496] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.912500] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.912504] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.912510] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.912516] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.912520] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.912522] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.912526] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.912531] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.912536] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.912539] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.912542] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.912546] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] query failed(ret=-5066, conn=0x2b399eda8060, start=1719306018911109, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.912551] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.912569] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.912574] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.912579] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.912584] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.912587] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.912590] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.912596] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:4000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.912600] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.912606] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.914326] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [100337][T1003_Occam][T1003][Y0-0000000000000000-0-0] [lt=13] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-06-25 17:00:18.914596] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3790385, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=203, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:18.916811] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.916820] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.916825] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:18.916832] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.916838] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.922638] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [62718][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=12] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=969588866, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=408742, mem_used=767425536) [2024-06-25 17:00:18.929638] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:18.929673] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT GROUP LOG INFO](palf_id=1005, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:18.929682] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:18.929686] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:18.929688] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:18.929691] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:18.929912] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164325962AB4-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301546, send_timestamp=1719305996628365) [2024-06-25 17:00:18.931095] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-000616430B73EF50-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=330, fly_ts=22301542, send_timestamp=1719305996629553) [2024-06-25 17:00:18.931153] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=2) [2024-06-25 17:00:18.932467] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:18.933468] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1211) [2024-06-25 17:00:18.937976] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:18.938306] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [63346][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=12] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-06-25 17:00:18.939409] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [42764][T1002_ComTimerT][T1002][Y0-0000000000000000-0-0] [lt=9] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-06-25 17:00:18.940594] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1A-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:18.945321] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [63469][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=4] ====== checkpoint timer task ====== [2024-06-25 17:00:18.945332] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:193) [63469][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=8] no logstream(ret=0, ls_cnt=0) [2024-06-25 17:00:18.945488] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.948088] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [100333][T1003_Occam][T1003][YB420ABF4533-000616615EBF73AC-0-0] [lt=9] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:18.952106] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=77000, remain_us=22872872, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=77, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:18.955422] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [62818][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=7] current monitor number(seq_=-1) [2024-06-25 17:00:18.958456] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] ====== checkpoint timer task ====== [2024-06-25 17:00:18.958938] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] succ to get rec scn(*this={this:0x2b3aae604030, ls_id:{id:1006}, tenant_id:1002, state:"L_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171811467, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:151}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.958954] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=16] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb4080, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171812500, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb4080, snapshot_version:{val:1718209171812500, v:0}, ls_id:{id:1006}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.958974] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=15] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1006}) [2024-06-25 17:00:18.958997] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=9] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3a9534a510, is_inited:true, freezing_scn:{val:1718285174571744002, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1006}, ls_create_status:1, clog_checkpoint_scn:{val:1719306013505123002, v:0}, clog_base_lsn:{lsn:75425759232}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315392841000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:1, proposal_id:1, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1006}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1006, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:110, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:18.959030] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=31] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1006}) [2024-06-25 17:00:18.959045] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] get_binary_search_range_(ret=0, this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1112, max_block_id=1124, result_block_id=274894685184, committed_lsn={lsn:75469139099}, scn={val:1719306018752321001, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:18.959587] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=12] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759415, offset=21479848}, end_cursor:ObLogCursor{file_id=67, log_id=18759416, offset=21480089}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb8c050, len:3672, log_data_len:241, seq:18759415, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:18.959605] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:75425759232}, checkpoint_scn={val:1719306018752321001, v:0}, ls_id={id:1006}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:18.959617] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:75425759232}, new_base_lsn={lsn:75425759232}, lsn={lsn:75425759232}) [2024-06-25 17:00:18.959625] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:75425759232}, ls->get_ls_id()={id:1006}) [2024-06-25 17:00:18.959958] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325C300CB-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301544, send_timestamp=1719305996658414) [2024-06-25 17:00:18.959982] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C300CB-0-0] [lt=5] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf16f470, extra_info="") [2024-06-25 17:00:18.960081] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=79, this=0x2b3aaf1793b0, extra_info="palf_id:1010") [2024-06-25 17:00:18.960095] INFO [LIB] stat (utility.h:1214) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=5, this=0x2b3a8ce1d3b0, extra_info="") [2024-06-25 17:00:18.960094] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] succ to get rec scn(*this={this:0x2b3aae6b6030, ls_id:{id:1007}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171811972, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:149}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.960102] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb4e90, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171813010, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb4e90, snapshot_version:{val:1718209171813010, v:0}, ls_id:{id:1007}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.960106] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=6, this=0x2b3aaf1795e0, extra_info="palf_id:1010") [2024-06-25 17:00:18.960111] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1007}) [2024-06-25 17:00:18.960130] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3a86ce4510, is_inited:true, freezing_scn:{val:1718289975940695002, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1007}, ls_create_status:1, clog_checkpoint_scn:{val:1719306013868334001, v:0}, clog_base_lsn:{lsn:85155950592}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315408583001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1007}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1007, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:18.960154] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=23] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1007}) [2024-06-25 17:00:18.960161] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] get_binary_search_range_(ret=0, this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1257, max_block_id=1269, result_block_id=274894685184, committed_lsn={lsn:85187186692}, scn={val:1719306018870127000, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:18.960287] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC8D5-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf16f6a0, extra_info="palf_id:1010") [2024-06-25 17:00:18.960292] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC8D5-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=295, this=0x2b3aaf16f8d0, extra_info="palf_id:1010") [2024-06-25 17:00:18.960635] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=12] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759416, offset=21480089}, end_cursor:ObLogCursor{file_id=67, log_id=18759417, offset=21480330}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb98050, len:3431, log_data_len:241, seq:18759416, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:18.960646] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:85155950592}, checkpoint_scn={val:1719306018870127000, v:0}, ls_id={id:1007}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:18.960653] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:85155950592}, new_base_lsn={lsn:85155950592}, lsn={lsn:85155950592}) [2024-06-25 17:00:18.960659] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:85155950592}, ls->get_ls_id()={id:1007}) [2024-06-25 17:00:18.961092] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62241][pnio1][T0][YB420ABF4533-0006166171AF65D4-0-0] [lt=7][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:18.961103] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62241][pnio1][T0][YB420ABF4533-0006166171AF65D4-0-0] [lt=11] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:18.961120] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62241][pnio1][T0][YB420ABF4533-0006166171AF65D4-0-0] [lt=8] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306018960742]}) [2024-06-25 17:00:18.961126] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=3] succ to get rec scn(*this={this:0x2b3ab7604030, ls_id:{id:1008}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629001, v:0}, last_push_gc_task_ts_:1718209171812477, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:149}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.961134] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb5ca0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171815569, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb5ca0, snapshot_version:{val:1718209171815569, v:0}, ls_id:{id:1008}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.961144] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=9] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1008}) [2024-06-25 17:00:18.961164] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3a86daa510, is_inited:true, freezing_scn:{val:1718286006014126001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1008}, ls_create_status:1, clog_checkpoint_scn:{val:1719306013537772003, v:0}, clog_base_lsn:{lsn:74083663872}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315055094004, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1008}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1008, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-550728704, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:18.961186] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=22] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1008}) [2024-06-25 17:00:18.961192] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] get_binary_search_range_(ret=0, this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1092, max_block_id=1104, result_block_id=274894685184, committed_lsn={lsn:74143917071}, scn={val:1719306018543922003, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:18.961690] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759417, offset=21480330}, end_cursor:ObLogCursor{file_id=67, log_id=18759418, offset=21480571}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb7c050, len:3190, log_data_len:241, seq:18759417, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:18.961702] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:74083663872}, checkpoint_scn={val:1719306018543922003, v:0}, ls_id={id:1008}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:18.961716] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:74083663872}, new_base_lsn={lsn:74083663872}, lsn={lsn:74083663872}) [2024-06-25 17:00:18.961724] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:74083663872}, ls->get_ls_id()={id:1008}) [2024-06-25 17:00:18.962197] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] succ to get rec scn(*this={this:0x2b3ab76b6030, ls_id:{id:1009}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171812974, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:152}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.962207] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb6ab0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171817146, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb6ab0, snapshot_version:{val:1718209171817146, v:0}, ls_id:{id:1009}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.962221] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=11] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1009}) [2024-06-25 17:00:18.962247] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3aaf086510, is_inited:true, freezing_scn:{val:1718283438009682001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1009}, ls_create_status:1, clog_checkpoint_scn:{val:1719306013750401001, v:0}, clog_base_lsn:{lsn:86632255488}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315224596003, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1009}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1009, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:18.962276] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=27] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1009}) [2024-06-25 17:00:18.962286] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] get_binary_search_range_(ret=0, this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1279, max_block_id=1291, result_block_id=274894685184, committed_lsn={lsn:86651599708}, scn={val:1719306018805948000, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:18.962762] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759418, offset=21480571}, end_cursor:ObLogCursor{file_id=67, log_id=18759419, offset=21480812}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb88050, len:2949, log_data_len:241, seq:18759418, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:18.962772] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:86632255488}, checkpoint_scn={val:1719306018805948000, v:0}, ls_id={id:1009}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:18.962780] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:86632255488}, new_base_lsn={lsn:86632255488}, lsn={lsn:86632255488}) [2024-06-25 17:00:18.962786] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:86632255488}, ls->get_ls_id()={id:1009}) [2024-06-25 17:00:18.963257] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=3] succ to get rec scn(*this={this:0x2b3a84404030, ls_id:{id:1010}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171813475, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:151}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.963265] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb78c0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171818018, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb78c0, snapshot_version:{val:1718209171818018, v:0}, ls_id:{id:1010}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.963276] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=9] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1010}) [2024-06-25 17:00:18.963294] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3aaf132510, is_inited:true, freezing_scn:{val:1718283437110618003, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1010}, ls_create_status:1, clog_checkpoint_scn:{val:1719198894656674000, v:0}, clog_base_lsn:{lsn:75627073536}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315483366000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1010}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1010, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:18.963319] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=23] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x2b3aaf1321b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x2b3aaf137cd0, table_type:1, memtable_head:1774, memtable_tail:1775, t3m:0x2b3a8a6b2030, tables:[null, null, null, null, null, null, null, null, null, null, null, null, null, null, 0x2b3a86c90080, null]}, is_freezing:false, ls_id:{id:1010}, mini_merge_recycle_commit_versions_ts:1719188582135344, tx_data_table:0x2b3aaf1391d0, ls_tablet_svr:0x2b3aaf132190, slice_allocator:0x2b3aaf139210}, min_rec_scn={val:1719198894656674000, v:0}, ls_id_={id:1010}) [2024-06-25 17:00:18.963334] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=13] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1719198894656674000, v:0}, checkpoint_scn_in_ls_meta={val:1719198894656674000, v:0}, ls_id={id:1010}, service_type="TRANS_SERVICE") [2024-06-25 17:00:18.963341] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1010, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:75627073536}, new_base_lsn={lsn:75627073536}, lsn={lsn:75627073536}) [2024-06-25 17:00:18.963346] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:75627073536}, ls->get_ls_id()={id:1010}) [2024-06-25 17:00:18.963350] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:191) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] succeed to update_clog_checkpoint(ret=0, ls_cnt=5) [2024-06-25 17:00:18.966211] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [62616][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=7] ====== tenant freeze timer task ====== [2024-06-25 17:00:18.966276] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:18.966286] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:18.966289] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:18.966291] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:18.966294] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:18.966514] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-000616432547EFF3-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301556, send_timestamp=1719305996664957) [2024-06-25 17:00:18.974085] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=12] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289109524480, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:18.975028] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [99158][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=29] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}, {1001,0,False,{False,},False,False,True}, {1002,1,False,{False,},False,False,True}, {1003,1,False,{False,},False,False,True}, {1004,1,False,{False,},False,False,True}, {1005,0,False,{False,},False,False,True}, {1006,1,False,{False,},False,False,True}, {1007,1,False,{False,},False,False,True}, {1008,1,False,{False,},False,False,True}, {1039,0,False,{False,},False,False,True}]) [2024-06-25 17:00:18.977251] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62244][pnio1][T1001][YB420ABF4533-00061661718F6531-0-0] [lt=10][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:18.985113] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.002097] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SYS](tenant_id=1, io_status=sys_group_name: MICRO_BLOCK_CACHE_IO, mode: read, size: 4096.00, iops: 11.19, rt: 76.00) [2024-06-25 17:00:19.002108] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS CONFIG](tenant_id=1, ref_cnt=2, io_config={group_num_:0, memory_limit_:8589934592, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:4}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=12763552, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:19.002142] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 0.93, rt: 105.00) [2024-06-25 17:00:19.002148] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SYS](tenant_id=500, io_status=sys_group_name: INSPECT_BAD_BLOCK_IO, mode: read, size: 2097152.00, iops: 0.93, rt: 775.00) [2024-06-25 17:00:19.002152] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:19.002188] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.73, rt: 68.25) [2024-06-25 17:00:19.002193] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:1073741824, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:1}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10735904, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:19.002219] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SYS](tenant_id=1004, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 4.66, rt: 66.80) [2024-06-25 17:00:19.002224] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=1004, ref_cnt=2, io_config={group_num_:0, memory_limit_:724775731200, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:100}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=12763552, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:19.002236] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002244] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002246] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002250] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002255] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002257] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002261] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002264] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002268] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002271] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002274] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002278] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002281] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002284] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002288] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002291] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.009756] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=9939511, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=163, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.010125] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:19.010134] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=8][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:19.010143] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=9][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:19.010146] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:19.010150] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:19.010155] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.010160] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:19.010166] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:19.010174] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:19.010178] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:19.010180] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:19.010186] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:19.010192] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:19.010195] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.010201] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:19.010204] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:19.010208] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] query failed(ret=-5066, conn=0x2b3b6150d470, start=1719306019008713, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:19.010215] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.010240] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:19.010245] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.010251] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.010257] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.010261] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.010264] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.010271] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.010276] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.010282] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:19.010955] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5634385, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=191, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.013808] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=53000, remain_us=25713368, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=53, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.020588] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.020598] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:19.020604] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:19.020611] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.020616] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.022811] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.022824] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.022843] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=1] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e6b6780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221534415, assoc_session_id:3221534415, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306018751663, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3a97f681f8, id:3221534415, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e6b6780}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:19.025313] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.025322] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.025327] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:19.025333] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.025337] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.025962] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AB7-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=97, avg_log_submit_to_slide_time=845) [2024-06-25 17:00:19.025971] INFO [PALF] read (log_cache.cpp:100) [99518][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.027404] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63474][T1001_LSArchive][T1001][YB420ABF4533-000616423F8E6725-0-0] [lt=9] gc stale ls task succ [2024-06-25 17:00:19.029711] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT GROUP LOG INFO](palf_id=1039, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:19.029720] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99931][T1004_L0_G0][T1004][YB420ABA45B1-000616460A03D1DF-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.029724] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=12] [PALF STAT APPEND DATA SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1211) [2024-06-25 17:00:19.029734] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.029767] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT APPEND DATA SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1211) [2024-06-25 17:00:19.029773] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.029776] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.029779] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.029783] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.029841] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1333) [2024-06-25 17:00:19.029955] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABDC060-0006164325962AB8-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301540, send_timestamp=1719305996728415) [2024-06-25 17:00:19.029993] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325962AB9-0-0] [lt=0][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301538, send_timestamp=1719305996728454) [2024-06-25 17:00:19.031180] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.031189] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.031193] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.031199] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.031204] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.032370] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4383, dropped:1103, tid:62275}]) [2024-06-25 17:00:19.032493] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.032930] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [100318][T1004_L0_G0][T1004][YB420ABF4533-00061660EF5FD0A2-0-0] [lt=8] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1719305953432726, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1719306019032157, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-06-25 17:00:19.032982] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [100318][T1004_L0_G0][T1004][YB420ABF4533-00061660EF5FD0A2-0-0] [lt=13] current clock is (clock_=0, max_seq_=0, mem_limit=161956757504, cur_mem_hold=55475961856, attr_.tenant_id_=1004) [2024-06-25 17:00:19.034130] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100263][T1004_L0_G0][T1004][YB420ABA45B1-000616460A03D1DF-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.038032] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.041050] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.041058] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.041062] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.041068] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.041072] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.050141] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFF5-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1333) [2024-06-25 17:00:19.052327] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.052337] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.052341] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78dd0d80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306018997478, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.052359] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78dd0d80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306018997478, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.052371] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.052377] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.052380] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.052384] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.052386] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.052389] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.052393] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=211, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.052398] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=2556396, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=211, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.052799] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [99392][T1004_ReqMemEvi][T1004][Y0-0000000000000000-0-0] [lt=11] sql audit evict task end(request_manager_->get_tenant_id()=1004, evict_high_mem_level=20669530113, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=8345564, mem_used=15148855296) [2024-06-25 17:00:19.053460] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63292][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=8] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7268, mem_used=12478464) [2024-06-25 17:00:19.054680] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.054689] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.054692] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a69360780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019000641, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.054709] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a69360780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019000641, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.054720] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.054725] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.054728] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.054731] WDIAG [SQL] open (ob_result_set.cpp:157) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.054733] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.054737] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.054741] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=132, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.054746] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4] will sleep(sleep_us=100000, remain_us=14680077, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=132, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.054772] INFO [LIB] stat (utility.h:1214) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbfec70, extra_info="") [2024-06-25 17:00:19.057241] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.057252] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.057256] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb219e9e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019002748, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.057279] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=23][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb219e9e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019002748, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.057291] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.057299] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.057304] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.057307] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.057312] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.057315] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.057321] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=215, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.057327] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=1918563, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=215, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.058856] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.058867] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.060024] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164325C300CF-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301545, send_timestamp=1719305996758478) [2024-06-25 17:00:19.060313] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABF4533-00061661728FC8D9-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301546, send_timestamp=1719305996758767) [2024-06-25 17:00:19.062046] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B35-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1009, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=9, avg_log_batch_cnt=0, total_group_log_size=1098, avg_group_log_size=122) [2024-06-25 17:00:19.062911] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFF6-0-0] [lt=9] [PALF STAT LOG TASK TIME](palf_id=1009, self="10.191.69.51:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=91, avg_log_submit_to_slide_time=842) [2024-06-25 17:00:19.062916] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT HOT CACHE HIT RATE](palf_id=1009, read_size=1098, hit_cnt=9, read_cnt=9, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.066327] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.066335] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.066338] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.066340] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.066342] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.066574] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-000616432547EFF7-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301550, send_timestamp=1719305996765024) [2024-06-25 17:00:19.066872] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=9][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:53, local_retry_times:53, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.066878] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=6][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.066881] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.066883] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.066888] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.068297] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.068304] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=7][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.068306] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e68be20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019014313, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.068319] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=13][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e68be20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019014313, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.068329] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=10][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.068334] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.068338] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.068340] WDIAG [SQL] open (ob_result_set.cpp:157) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.068342] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.068345] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.068349] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=204, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.068352] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3] will sleep(sleep_us=100000, remain_us=3636629, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=204, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.075331] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.075344] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.075352] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:19.075359] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:19.075362] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.075366] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:19.075369] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.075371] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.075375] WDIAG [SQL.ENG] inner_get_next_batch_for_tsc (ob_table_scan_op.cpp:2015) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:19.075377] WDIAG [SQL.ENG] inner_get_next_batch (ob_table_scan_op.cpp:2000) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:19.075380] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] get next batch failed(ret=-4383, eval_ctx={batch_idx:0, batch_size:16, max_batch_size:16, frames_:0x2bb27d012e10}, id=2, op_name="PHY_TABLE_SCAN") [2024-06-25 17:00:19.075386] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1535) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4383] get next batch failed(ret=-4383) [2024-06-25 17:00:19.075388] WDIAG [SQL.ENG] get_next_row_vectorizely (ob_operator.cpp:1463) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] Failed to get_next_row vectorizedly(ret=-4383) [2024-06-25 17:00:19.075391] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_transmit_op.cpp:27) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] get next row from child failed(ret=-4383) [2024-06-25 17:00:19.075395] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4383] inner get next row failed(ret=-4383, type=14, op="PHY_DIRECT_TRANSMIT", op_id=1) [2024-06-25 17:00:19.075398] WDIAG [SQL.EXE] sync_send_result (ob_remote_executor_processor.cpp:283) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:19.075402] WDIAG [SQL.EXE] execute_remote_plan (ob_remote_executor_processor.cpp:469) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4383] sync send result failed(ret=-4383) [2024-06-25 17:00:19.075406] WDIAG [SQL.EXE] process (ob_remote_executor_processor.cpp:969) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] execute remote plan failed(ret=-4383, local_tenant_schema_version=1718209168713032, exec_ctx_.get_task_exec_ctx().get_query_tenant_begin_schema_version()=1718209168713032) [2024-06-25 17:00:19.077307] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62249][pnio1][T1001][YB420ABF4533-00061661718F6532-0-0] [lt=5][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.082951] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.082960] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.082964] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6790d360, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019028996, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.082981] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6790d360, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019028996, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.082992] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.082999] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.083002] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.083006] WDIAG [SQL] open (ob_result_set.cpp:157) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.083008] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.083011] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.083016] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=78, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.083022] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5] will sleep(sleep_us=78000, remain_us=22741957, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=78, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.085168] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.090439] INFO [COMMON] process (ob_rpc_processor_simple.cpp:2405) [95444][T1002_L0_G19][T1002][YB420ABDC060-000616431A273CD4-0-0] [lt=11] handle ObQueryLSIsValidMember requeset(arg_={tenant_id_:1002, self_addr_:"10.189.192.96:2882", ls_array_:[{id:1006}]}) [2024-06-25 17:00:19.090449] INFO [CLOG] get_member_gc_stat (ob_log_handler.cpp:1509) [95444][T1002_L0_G19][T1002][YB420ABDC060-000616431A273CD4-0-0] [lt=8] get_member_gc_stat(is_valid_member=true, stat=1, member_list=5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, learner_list={learner_num:0, learner_array:[]}, addr="10.189.192.96:2882", this={role:1, proposal_id:1, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1006}) [2024-06-25 17:00:19.092544] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.104195] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC07C-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1, self="10.191.69.51:2882", committed size=4306) [2024-06-25 17:00:19.105015] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A081-0-0] [lt=10] [PALF STAT LOG TASK TIME](palf_id=1, self="10.191.69.51:2882", total_slide_log_cnt=12, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=93, avg_log_submit_to_slide_time=829) [2024-06-25 17:00:19.105023] INFO [PALF] read (log_cache.cpp:100) [62747][T1_ReplaySrv4][T1][Y0-0000000000000000-0-0] [lt=3] [PALF STAT HOT CACHE HIT RATE](palf_id=1, read_size=3889, hit_cnt=12, read_cnt=12, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.108666] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=9] new mysql sessid created(conn.sessid_=3221510720, support_ssl=false) [2024-06-25 17:00:19.108684] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=8] sm conn init succ(conn.sessid_=3221510720, sess.client_addr_="10.191.69.51:47998") [2024-06-25 17:00:19.108691] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=5] accept one succ(*s={this:0x2b3ac6e24d70, session_id:3221510720, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:907}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:19.108884] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x2b3ac6e24d70, session_id:3221510720, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:907}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:19.108893] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=8] kill and revert session(conn.sessid_=3221510720, proxy_sessid=0, server_id=1, ret=0) [2024-06-25 17:00:19.108898] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:918) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=5] sock ref clean, do destroy(*s={this:0x2b3ac6e24d70, session_id:3221510720, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:907}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:19.108905] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:223) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=6] mark session id unused(conn.sessid_=3221510720) [2024-06-25 17:00:19.108908] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=2] connection close(sessid=3221510720, proxy_sessid=0, tenant_id=0, server_id=1, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=false, ret=0, trace_id=Y0-0000000000000000-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:0], disconnect_state=0) [2024-06-25 17:00:19.109673] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=7] re_register_parent reason(palf_id=1006, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306015007500, last_first_register_time_us=1713332560377970, register_time_us=1719306015007500, curr_time_us=1719306019109671) [2024-06-25 17:00:19.109697] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=11] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1006, self="10.191.69.51:2882", reason=1, leader="10.191.69.56:2882", register_time_us=1719306019109685, last_submit_register_req_time_us=1719306019109685, register_parent_reason=1) [2024-06-25 17:00:19.109706] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=8] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1006, self="10.191.69.51:2882") [2024-06-25 17:00:19.109822] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=9][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:163, local_retry_times:163, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.109836] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=11][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.109840] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.109843] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.109851] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.111022] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=9][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:191, local_retry_times:191, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.111033] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.111037] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.111040] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.111046] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.120716] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.120728] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.120732] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6d6bf4a0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019066854, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.120750] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=18][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6d6bf4a0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019066854, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.120761] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.120768] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.120772] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.120774] WDIAG [SQL] open (ob_result_set.cpp:157) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.120777] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.120780] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.120784] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=54, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.120790] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5] will sleep(sleep_us=54000, remain_us=25606386, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=54, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.124105] WDIAG [STORAGE.TRANS] process_ (ob_gti_rpc.h:178) [62242][pnio1][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=8][errcode=0] gti rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:19.125184] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBADE-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:19.125410] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC89D-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:19.126255] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFF8-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=91, avg_log_submit_to_slide_time=837) [2024-06-25 17:00:19.129763] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.129786] INFO eloop_run (eloop.c:143) [62232][pnio1][T0][Y0-0000000000000000-0-0] [lt=13] PNIO [ratelimit] time: 1719306019129786, bytes: 29137865946318, bw: 0.739710 MB/s, add_ts: 1000191, add_bytes: 775790 [2024-06-25 17:00:19.129799] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.129802] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.129806] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.129810] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.129998] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325962ABD-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301540, send_timestamp=1719305996828457) [2024-06-25 17:00:19.130038] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-0006164325962ABE-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301556, send_timestamp=1719305996828482) [2024-06-25 17:00:19.131221] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=10] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=1) [2024-06-25 17:00:19.132550] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.132635] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301554, send_timestamp=1719305996831080) [2024-06-25 17:00:19.133399] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF59C7-0-0] [lt=8] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b61c69470, extra_info="") [2024-06-25 17:00:19.133444] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.133475] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=64, this=0x2b3b61c733b0, extra_info="palf_id:1006") [2024-06-25 17:00:19.133504] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3b61c735e0, extra_info="palf_id:1006") [2024-06-25 17:00:19.133690] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ABF-0-0] [lt=6] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b61c696a0, extra_info="palf_id:1006") [2024-06-25 17:00:19.133694] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ABF-0-0] [lt=4] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=280, this=0x2b3b61c698d0, extra_info="palf_id:1006") [2024-06-25 17:00:19.137448] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.138089] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.138493] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABF4538-000616694DDF523B-0-0] [lt=9] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1006}}) [2024-06-25 17:00:19.138508] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABF4538-000616694DDF523B-0-0] [lt=7] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306018870127002, v:0}, arg={tenant_id:1004, ls_id:{id:1006}}, result={tenant_id:1004, ls_id:{id:1006}, cur_readable_scn:{val:1719306018870127002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:19.140924] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] [2024-06-25 17:00:19.140931] WDIAG [SQL.EXE] setup_next_scanner (ob_direct_receive_op.cpp:207) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4012] error occurring in the remote sql execution, please use the current TRACE_ID to grep the original error message on the remote_addr.(ret=-4012, remote_addr="10.186.69.171:2882") [2024-06-25 17:00:19.140936] WDIAG [SQL.EXE] inner_open (ob_direct_receive_op.cpp:53) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] failed to setup first scanner(ret=-4012) [2024-06-25 17:00:19.140940] WDIAG [SQL.ENG] open (ob_operator.cpp:714) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] Open this operator failed(ret=-4012, op_type="PHY_DIRECT_RECEIVE") [2024-06-25 17:00:19.140946] WDIAG [SQL.EXE] open (ob_execute_result.cpp:104) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] open operator failed(ret=-4012) [2024-06-25 17:00:19.140949] WDIAG [SQL] open_result (ob_result_set.cpp:210) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail open main query(ret=-4012) [2024-06-25 17:00:19.140954] WDIAG [SQL] open (ob_result_set.cpp:159) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.140956] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.140959] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.140965] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] execute failed(ret=-4012, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:19.140971] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:19.140978] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.140981] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.140984] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:19.140990] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:19.140996] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4012, ret=-4012) [2024-06-25 17:00:19.140999] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.141004] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1) [2024-06-25 17:00:19.141008] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:19.141013] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] query failed(ret=-4012, conn=0x2b3a7c6db470, start=1719306019138350, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:19.141018] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.141040] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] get all ls info by persistent_ls_ failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1001) [2024-06-25 17:00:19.141046] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1247) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] the sql used to get ls locations error, treat as timeout(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", previous_ret=-4012, ls_ids=[{id:1}]) [2024-06-25 17:00:19.141054] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4664] batch renew ls locations failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.141059] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] renew location failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.141063] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get location(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.141067] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get log stream location leader(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.141074] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4664] get leader failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:3000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.141078] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.141084] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4664] fetch new range failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT") [2024-06-25 17:00:19.141087] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:19.141093] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:19.141096] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.141101] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:19.141104] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.141106] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.141112] WDIAG [SQL.ENG] inner_get_next_row_for_tsc (ob_table_scan_op.cpp:1913) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:19.141115] WDIAG [SQL.ENG] inner_get_next_row_implement (ob_table_scan_op.cpp:1900) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:19.141118] WDIAG [SQL.ENG] inner_get_next_row (ob_table_scan_op.cpp:2861) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] get next row failed(ret=-4383) [2024-06-25 17:00:19.141123] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4383] inner get next row failed(ret=-4383, type=3, op="PHY_TABLE_SCAN", op_id=0) [2024-06-25 17:00:19.141127] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4383] get next row from operator failed(ret=-4383) [2024-06-25 17:00:19.141131] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] get next row from exec result failed(ret=-4383) [2024-06-25 17:00:19.141135] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4383] get_next_row failed(ret=-4383, has_tenant_resource()=true) [2024-06-25 17:00:19.141138] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')"}) [2024-06-25 17:00:19.141142] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1002, executor={ObIExecutor:, sql:"SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')"}, retry_cnt=34, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1718209168713032) [2024-06-25 17:00:19.141146] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] will sleep(sleep_us=34000, remain_us=58992, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=34, v.err_=-4383, timeout_timestamp=1719306019200138) [2024-06-25 17:00:19.141862] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62415][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142153] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62429][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142245] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62426][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142288] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62430][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142414] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62416][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142632] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62420][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142634] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62432][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142653] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62425][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142698] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62417][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142713] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62421][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142760] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62423][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142869] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62422][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142984] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62431][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.143007] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62419][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.143017] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62418][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.143087] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62424][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.143140] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62427][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.143470] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62428][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.145353] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.149115] INFO [STORAGE.BLKMGR] inspect_bad_block (ob_block_manager.cpp:1712) [62325][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=13] check macro block(block_info={ref_cnt:1, is_free:false, access_time:1717170479939398}, time_interval=2135539209716) [2024-06-25 17:00:19.149839] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF78EC-0-0] [lt=5] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86d1d470, extra_info="") [2024-06-25 17:00:19.149850] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF78EC-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1333, avg_group_log_size=121) [2024-06-25 17:00:19.149931] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=81, this=0x2b3a86d273b0, extra_info="palf_id:1007") [2024-06-25 17:00:19.149951] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a86d275e0, extra_info="palf_id:1007") [2024-06-25 17:00:19.149997] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:19.150134] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFF9-0-0] [lt=4] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a86d1d6a0, extra_info="palf_id:1007") [2024-06-25 17:00:19.150140] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFF9-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=287, this=0x2b3a86d1d8d0, extra_info="palf_id:1007") [2024-06-25 17:00:19.152463] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:211, local_retry_times:211, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.152473] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.152476] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.152479] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.152485] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.154744] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABE7C-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.154830] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:132, local_retry_times:132, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.154841] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.154846] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.154848] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.154854] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.155179] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.157391] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=7][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:215, local_retry_times:215, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.157399] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=7][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.157402] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.157405] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.157411] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.158465] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.158477] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.160076] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325C300D3-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301548, send_timestamp=1719305996858527) [2024-06-25 17:00:19.161082] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:78, local_retry_times:78, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.161092] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.161095] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.161098] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.161104] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.163486] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=7][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.163495] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.163498] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e775ba0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019108899, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.163520] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=22][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e775ba0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019108899, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.163532] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.163539] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.163543] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.163546] WDIAG [SQL] open (ob_result_set.cpp:157) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.163549] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.163553] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.163557] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=164, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.163563] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=9785704, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=164, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.164655] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.164665] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.164669] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6ab5c780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019111013, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.164689] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=20][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6ab5c780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019111013, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.164701] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999998, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.164707] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.164712] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.164714] WDIAG [SQL] open (ob_result_set.cpp:157) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.164718] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.164721] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}) [2024-06-25 17:00:19.164726] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, retry_cnt=192, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.164732] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=5480609, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=192, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.166386] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.166392] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.166395] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.166397] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.166400] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.166475] INFO [LIB] stat (utility.h:1214) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3a8ce1cc70, extra_info="") [2024-06-25 17:00:19.167067] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [63199][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=9] ====== tenant freeze timer task ====== [2024-06-25 17:00:19.168410] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:204, local_retry_times:204, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.168418] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=7][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.168420] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.168423] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.168429] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.172291] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=0][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:19.172299] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:19.172876] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.173982] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63309][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=7] start do ls ha handler(ls_id_array_=[{id:1010}, {id:1008}, {id:1009}, {id:1006}, {id:1007}]) [2024-06-25 17:00:19.174339] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289111621632, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:19.174851] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:54, local_retry_times:54, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.174862] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.174865] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.174868] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.174875] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.175206] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:34, local_retry_times:34, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.175215] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.175218] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.175220] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.175630] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.177325] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62238][pnio1][T1001][YB420ABF4533-00061661718F6533-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.178777] INFO [STORAGE] runTimerTask (ob_locality_manager.cpp:708) [62328][LocalityReload][T0][Y0-0000000000000000-0-0] [lt=7] runTimer to refresh locality_info(ret=0) [2024-06-25 17:00:19.179204] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-4012] [2024-06-25 17:00:19.179210] WDIAG [SQL.EXE] setup_next_scanner (ob_direct_receive_op.cpp:207) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] error occurring in the remote sql execution, please use the current TRACE_ID to grep the original error message on the remote_addr.(ret=-4012, remote_addr="10.186.69.171:2882") [2024-06-25 17:00:19.179214] WDIAG [SQL.EXE] inner_open (ob_direct_receive_op.cpp:53) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] failed to setup first scanner(ret=-4012) [2024-06-25 17:00:19.179218] WDIAG [SQL.ENG] open (ob_operator.cpp:714) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] Open this operator failed(ret=-4012, op_type="PHY_DIRECT_RECEIVE") [2024-06-25 17:00:19.179222] WDIAG [SQL.EXE] open (ob_execute_result.cpp:104) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] open operator failed(ret=-4012) [2024-06-25 17:00:19.179225] WDIAG [SQL] open_result (ob_result_set.cpp:210) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] fail open main query(ret=-4012) [2024-06-25 17:00:19.179227] WDIAG [SQL] open (ob_result_set.cpp:159) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.179235] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.179239] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.179242] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute failed(ret=-4012, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:19.179248] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:19.179254] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.179258] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.179260] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:19.179264] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:19.179270] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4012, ret=-4012) [2024-06-25 17:00:19.179273] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.179277] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1) [2024-06-25 17:00:19.179281] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:19.179284] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] query failed(ret=-4012, conn=0x2b3ac92dd750, start=1719306019176694, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:19.179290] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.179310] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] get all ls info by persistent_ls_ failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1001) [2024-06-25 17:00:19.179315] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1247) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4664] the sql used to get ls locations error, treat as timeout(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", previous_ret=-4012, ls_ids=[{id:1}]) [2024-06-25 17:00:19.179319] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4664] batch renew ls locations failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.179324] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] renew location failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.179328] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get location(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.179331] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get log stream location leader(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.179337] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4664] get leader failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.179341] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.179346] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] fetch new range failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT") [2024-06-25 17:00:19.180321] INFO [SHARE] set_server_locality_array (ob_server_locality_cache.cpp:346) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=8] set_server_locality_array success(server_locality_array=[{inited:true, addr:"10.186.69.171:2882", zone:"zone3", zone_type:0, idc:"wg_sh16", region:"sys_region", is_active:true, start_service_time:1713323605265064, server_stop_time:0, server_status:1}, {inited:true, addr:"10.186.69.173:2882", zone:"zone3", zone_type:0, idc:"wg_sh16", region:"sys_region", is_active:true, start_service_time:1713323615484318, server_stop_time:0, server_status:1}, {inited:true, addr:"10.186.69.176:2882", zone:"zone4", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323605225767, server_stop_time:0, server_status:1}, {inited:true, addr:"10.186.69.177:2882", zone:"zone4", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323615512327, server_stop_time:0, server_status:1}, {inited:true, addr:"10.189.192.96:2882", zone:"zone5", zone_type:0, idc:"wg_88", region:"sys_region", is_active:true, start_service_time:1713323605377346, server_stop_time:0, server_status:1}, {inited:true, addr:"10.189.192.97:2882", zone:"zone5", zone_type:0, idc:"wg_88", region:"sys_region", is_active:true, start_service_time:1713323615580956, server_stop_time:0, server_status:1}, {inited:true, addr:"10.191.69.51:2882", zone:"zone1", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323604978888, server_stop_time:0, server_status:1}, {inited:true, addr:"10.191.69.52:2882", zone:"zone1", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323615397450, server_stop_time:0, server_status:1}, {inited:true, addr:"10.191.69.56:2882", zone:"zone2", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323605233519, server_stop_time:0, server_status:1}, {inited:true, addr:"10.191.69.57:2882", zone:"zone2", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323615481577, server_stop_time:0, server_status:1}], has_readonly_zone=false) [2024-06-25 17:00:19.180353] INFO [SHARE] load_region (ob_locality_table_operator.cpp:336) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=27] tenant_schema primary_zone is NULL, or no need calc region priority(tenant_id=1, primary_zone=RANDOM) [2024-06-25 17:00:19.180360] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=6] add_locality_zone success(locality_zone={tenant_id:1, region_priority:18446744073709551615}) [2024-06-25 17:00:19.180366] INFO [SHARE] load_region (ob_locality_table_operator.cpp:336) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=4] tenant_schema primary_zone is NULL, or no need calc region priority(tenant_id=1001, primary_zone=RANDOM) [2024-06-25 17:00:19.180370] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=3] add_locality_zone success(locality_zone={tenant_id:1001, region_priority:18446744073709551615}) [2024-06-25 17:00:19.180375] INFO [SHARE] load_region (ob_locality_table_operator.cpp:336) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=2] tenant_schema primary_zone is NULL, or no need calc region priority(tenant_id=1002, primary_zone=RANDOM) [2024-06-25 17:00:19.180379] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=3] add_locality_zone success(locality_zone={tenant_id:1002, region_priority:18446744073709551615}) [2024-06-25 17:00:19.180385] INFO [SHARE] load_region (ob_locality_table_operator.cpp:346) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=2] get_primary_zone_prioriry(primary_zone=zone1,zone2,zone3,zone4;zone5, tenant_primary_region_array=[{region:"sys_region", region_priority:0, zone_array:["zone1", "zone2", "zone3", "zone4", "zone5"]}], locality_info={version:0, local_region:"sys_region", local_zone:"zone1", local_idc:"wg_sh8", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone3", "zone4", "zone5", "zone1", "zone2"]}], locality_zone_array:[{tenant_id:1, region_priority:18446744073709551615}, {tenant_id:1001, region_priority:18446744073709551615}, {tenant_id:1002, region_priority:18446744073709551615}]}) [2024-06-25 17:00:19.180399] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=14] add_locality_zone success(locality_zone={tenant_id:1003, region_priority:0}) [2024-06-25 17:00:19.180403] INFO [SHARE] load_region (ob_locality_table_operator.cpp:346) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=2] get_primary_zone_prioriry(primary_zone=zone1,zone2,zone3,zone4;zone5, tenant_primary_region_array=[{region:"sys_region", region_priority:0, zone_array:["zone1", "zone2", "zone3", "zone4", "zone5"]}], locality_info={version:0, local_region:"sys_region", local_zone:"zone1", local_idc:"wg_sh8", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone3", "zone4", "zone5", "zone1", "zone2"]}], locality_zone_array:[{tenant_id:1, region_priority:18446744073709551615}, {tenant_id:1001, region_priority:18446744073709551615}, {tenant_id:1002, region_priority:18446744073709551615}, {tenant_id:1003, region_priority:0}]}) [2024-06-25 17:00:19.180411] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=8] add_locality_zone success(locality_zone={tenant_id:1004, region_priority:0}) [2024-06-25 17:00:19.180415] INFO [SHARE] load_region (ob_locality_table_operator.cpp:373) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=3] load region(ret=0, locality_info={version:0, local_region:"sys_region", local_zone:"zone1", local_idc:"wg_sh8", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone3", "zone4", "zone5", "zone1", "zone2"]}], locality_zone_array:[{tenant_id:1, region_priority:18446744073709551615}, {tenant_id:1001, region_priority:18446744073709551615}, {tenant_id:1002, region_priority:18446744073709551615}, {tenant_id:1003, region_priority:0}, {tenant_id:1004, region_priority:0}]}) [2024-06-25 17:00:19.181376] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.183586] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] [2024-06-25 17:00:19.183592] WDIAG [SQL.EXE] setup_next_scanner (ob_direct_receive_op.cpp:207) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] error occurring in the remote sql execution, please use the current TRACE_ID to grep the original error message on the remote_addr.(ret=-4012, remote_addr="10.186.69.171:2882") [2024-06-25 17:00:19.183596] WDIAG [SQL.EXE] inner_open (ob_direct_receive_op.cpp:53) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] failed to setup first scanner(ret=-4012) [2024-06-25 17:00:19.183599] WDIAG [SQL.ENG] open (ob_operator.cpp:714) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] Open this operator failed(ret=-4012, op_type="PHY_DIRECT_RECEIVE") [2024-06-25 17:00:19.183601] WDIAG [SQL.EXE] open (ob_execute_result.cpp:104) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] open operator failed(ret=-4012) [2024-06-25 17:00:19.183604] WDIAG [SQL] open_result (ob_result_set.cpp:210) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail open main query(ret=-4012) [2024-06-25 17:00:19.183607] WDIAG [SQL] open (ob_result_set.cpp:159) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.183610] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.183613] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.183618] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] execute failed(ret=-4012, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:19.183622] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:19.183628] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.183631] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.183633] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:19.183639] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:19.183644] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4012, ret=-4012) [2024-06-25 17:00:19.183648] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.183652] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1) [2024-06-25 17:00:19.183655] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:19.183658] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] query failed(ret=-4012, conn=0x2b3a78d44050, start=1719306019181411, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:19.183664] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.183679] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] get all ls info by persistent_ls_ failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1001) [2024-06-25 17:00:19.183684] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1247) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4664] the sql used to get ls locations error, treat as timeout(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", previous_ret=-4012, ls_ids=[{id:1}]) [2024-06-25 17:00:19.183687] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] batch renew ls locations failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.183692] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] renew location failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.183695] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get location(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.183699] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get log stream location leader(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.183705] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] get leader failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.183708] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3] fetch new DAS ID range finish(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.183714] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] fetch new range failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT") [2024-06-25 17:00:19.185224] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.189463] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.196314] WDIAG [RPC] send (ob_poc_rpc_proxy.h:156) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] sync rpc execute fail(ret=-4012, addr="10.186.69.171:2882", pcode=1308) [2024-06-25 17:00:19.196325] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=9][errcode=-4012] rpc task_execute fail(ret=-4012, tenant_id=1, svr="10.186.69.171:2882", timeout=4058, timeout_timestamp=1719306019200138) [2024-06-25 17:00:19.196331] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] task execute failed(ret=-4012) [2024-06-25 17:00:19.196335] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute with sql failed(ret=-4012) [2024-06-25 17:00:19.196338] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail execute plan(ret=-4012) [2024-06-25 17:00:19.196342] WDIAG [SQL] open (ob_result_set.cpp:157) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] execute plan failed(ret=-4012) [2024-06-25 17:00:19.196344] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.196347] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.196352] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] execute failed(ret=-4012, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:19.196357] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:19.196362] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.196366] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.196368] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:19.196373] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:19.196378] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4012, ret=-4012) [2024-06-25 17:00:19.196381] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.196385] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1) [2024-06-25 17:00:19.196389] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:19.196392] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] query failed(ret=-4012, conn=0x2b3a78d45a30, start=1719306019195828, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:19.196396] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.196412] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] get all ls info by persistent_ls_ failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1001) [2024-06-25 17:00:19.196417] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1247) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] the sql used to get ls locations error, treat as timeout(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", previous_ret=-4012, ls_ids=[{id:1}]) [2024-06-25 17:00:19.196421] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] batch renew ls locations failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.196424] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] renew location failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.196429] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] fail to get location(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.196432] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get log stream location leader(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.196438] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4664] get leader failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.196442] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.196447] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] fetch new range failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT") [2024-06-25 17:00:19.198301] INFO pktc_resp_cb_on_msg (pktc_resp.h:70) [62247][pnio1][T0][Y0-0000000000000000-0-0] [lt=7] PNIO resp cb not found: packet_id=1035876821 [2024-06-25 17:00:19.199313] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.199543] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=22] get_max_applied_scn(scn={val:1713323697228952000, v:0}, this={ls_id_:{id:1008}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713323697228952000, v:0}}) [2024-06-25 17:00:19.199557] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=12] get_min_unreplayed_log_info(lsn={lsn:74143917315}, scn={val:1719306019044617001, v:0}, this={ls_id_:{id:1008}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306019126252, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:74143917315}, next_to_submit_scn_:{val:1719306019044617001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713323697228952000, v:0}, iterator_:{iterator_impl:{buf_:0x2b3bda663000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:74143917193}, end_lsn:{lsn:74143917315}, read_buf:{buf_len_:73728, buf_:0x2b3bda663000}, block_size:67104768, log_storage_:0x2b3a86ded470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306019044617000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306019044617000, v:0}, data_checksum:3647337712, flag:0}}, init_mode_version:0, accumulate_checksum:2515280368, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306019044617000, v:0}}}}}) [2024-06-25 17:00:19.201253] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=5] LogLoopThread round_cost_time(us)(round_cost_time=13) [2024-06-25 17:00:19.206264] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.206273] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.206277] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78dd8400, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019152034, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.206294] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78dd8400, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019152034, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.206306] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.206313] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=6][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.206316] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.206319] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.206322] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.206326] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.206330] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=212, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.206336] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=2402459, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=212, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.206713] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.208634] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.208648] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=14526175, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=133, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.211087] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.211102] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1764788, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=216, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.212506] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1719306019200138, ctx_timeout_ts=-1, worker_timeout_ts=1719306019200138, default_timeout=1000000) [2024-06-25 17:00:19.212514] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.212517] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] batch renew ls locations failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.212521] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] renew location failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.212526] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] fail to get location(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.212529] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail to get log stream location leader(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.212534] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] get leader failed(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.212537] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2] fetch new DAS ID range finish(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.212541] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] fetch new range failed(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.214843] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.214857] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=79000, remain_us=22610122, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=79, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.215170] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=5] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647473, offset=60295969}, end_cursor:ObLogCursor{file_id=100, log_id=27647474, offset=60296210}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb80050, len:1247, log_data_len:241, seq:27647473, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:19.216290] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8012-0-0] [lt=9] push gts task success(*task={:""}) [2024-06-25 17:00:19.217124] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.218187] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8012-0-0] [lt=9] rpc statistics(total_trans_msg_count=2, total_batch_msg_count=0) [2024-06-25 17:00:19.218786] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=14] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647474, offset=60296210}, end_cursor:ObLogCursor{file_id=100, log_id=27647475, offset=60296459}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb90050, len:1006, log_data_len:249, seq:27647474, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:19.219814] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=4] ====== check clog disk timer task ====== [2024-06-25 17:00:19.219823] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=6] get_disk_usage(ret=0, capacity(MB):=32384, used(MB):=25883) [2024-06-25 17:00:19.219834] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=5] cannot_recycle_log_size statistics(cannot_recycle_log_size=229969377, threshold=10187125555, need_update_checkpoint_scn=false) [2024-06-25 17:00:19.221963] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3483019, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=205, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.223487] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647475, offset=60296459}, end_cursor:ObLogCursor{file_id=100, log_id=27647476, offset=60296708}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb9c050, len:757, log_data_len:249, seq:27647475, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:19.223601] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=5] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647476, offset=60296708}, end_cursor:ObLogCursor{file_id=100, log_id=27647477, offset=60296957}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fba0050, len:508, log_data_len:249, seq:27647476, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:19.223623] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=22] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:19.223745] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=5] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647477, offset=60296957}, end_cursor:ObLogCursor{file_id=100, log_id=27647478, offset=60297206}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb94050, len:4355, log_data_len:249, seq:27647477, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:19.225461] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC8A1-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86de7470, extra_info="") [2024-06-25 17:00:19.225550] INFO [LIB] stat (utility.h:1197) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=71, this=0x2b3a86df13b0, extra_info="palf_id:1008") [2024-06-25 17:00:19.225562] INFO [LIB] stat (utility.h:1214) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a8ce1daf0, extra_info="") [2024-06-25 17:00:19.225576] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a86df15e0, extra_info="palf_id:1008") [2024-06-25 17:00:19.226322] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFFC-0-0] [lt=6] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a86de76a0, extra_info="palf_id:1008") [2024-06-25 17:00:19.226327] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFFC-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=840, this=0x2b3a86de78d0, extra_info="palf_id:1008") [2024-06-25 17:00:19.228618] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.228635] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=55000, remain_us=25498541, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=55, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.229814] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.229847] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.229852] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.229854] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.229857] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.229890] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1333) [2024-06-25 17:00:19.230059] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164325962AC2-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305996928512) [2024-06-25 17:00:19.230082] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164325962AC3-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301540, send_timestamp=1719305996928542) [2024-06-25 17:00:19.232016] INFO eloop_run (eloop.c:143) [62250][pnio2][T0][Y0-0000000000000000-0-0] [lt=9] PNIO [ratelimit] time: 1719306019232016, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007622, add_bytes: 0 [2024-06-25 17:00:19.232575] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.233653] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AC4-0-0] [lt=6] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=79, avg_log_submit_to_slide_time=275) [2024-06-25 17:00:19.233662] INFO [PALF] read (log_cache.cpp:100) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=14] [PALF STAT HOT CACHE HIT RATE](palf_id=1006, read_size=1333, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.238144] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.244561] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1719306019200138, ctx_timeout_ts=-1, worker_timeout_ts=1719306019200138, default_timeout=1000000) [2024-06-25 17:00:19.244569] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=8][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.244573] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] batch renew ls locations failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.244580] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] renew location failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.244583] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail to get location(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.244587] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] fail to get log stream location leader(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.244594] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] get leader failed(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.244599] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.244604] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] fetch new range failed(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.244827] INFO [CLOG.EXTLOG] resize_log_ext_handler_ (ob_cdc_service.cpp:632) [62693][T1_CdcSrv][T1][Y0-0000000000000000-0-0] [lt=10] finish to resize log external storage handler(current_ts=1719306019244826, tenant_max_cpu=6, valid_ls_count=0, other_ls_count=0, new_concurrency=0) [2024-06-25 17:00:19.244906] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [99399][T1004_LSArchive][T1004][YB420ABF4533-00061642492E6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:19.250974] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.260125] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-0006164325C300D7-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301547, send_timestamp=1719305996958577) [2024-06-25 17:00:19.260408] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABF4538-00061669548FCBFC-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301542, send_timestamp=1719305996958866) [2024-06-25 17:00:19.264792] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.266440] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.266447] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.266450] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.266452] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.266455] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.266673] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-000616432547EFFE-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305996965128) [2024-06-25 17:00:19.272542] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.277268] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [63377][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=10] ====== tenant freeze timer task ====== [2024-06-25 17:00:19.277282] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [63377][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=9][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-06-25 17:00:19.277381] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62246][pnio1][T1001][YB420ABF4533-00061661718F6534-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.278548] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.284269] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061644F985525A-0-0] [lt=7] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1005, child={server:"10.186.69.176:2882", region:"sys_region", register_time_us:1719306019284181, keepalive_ts:-1}) [2024-06-25 17:00:19.284278] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061644F985525A-0-0] [lt=9] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1005, self="10.191.69.51:2882", child={server:"10.186.69.176:2882", region:"sys_region", register_time_us:1719306019284181, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:19.285283] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=13][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.288352] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.306396] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:212, local_retry_times:212, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.306407] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.306411] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.306413] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.306419] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.308666] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1719306019200138, ctx_timeout_ts=-1, worker_timeout_ts=1719306019200138, default_timeout=1000000) [2024-06-25 17:00:19.308673] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.308676] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] batch renew ls locations failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.308682] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] renew location failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.308685] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail to get location(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.308689] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail to get log stream location leader(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.308694] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] get leader failed(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.308698] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3] fetch new DAS ID range finish(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.308703] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] fetch new range failed(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.313370] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [62671][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=10] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-06-25 17:00:19.316726] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.317218] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=9632049, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=165, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.318531] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5326809, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=193, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.322028] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=10][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.325273] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBAE6-0-0] [lt=5] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b74f470, extra_info="") [2024-06-25 17:00:19.325365] INFO [LIB] stat (utility.h:1197) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=79, this=0x2b3b5b7593b0, extra_info="palf_id:1007") [2024-06-25 17:00:19.325374] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1211) [2024-06-25 17:00:19.325383] INFO [LIB] stat (utility.h:1214) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3afcbffaf0, extra_info="") [2024-06-25 17:00:19.325398] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x2b3b5b7595e0, extra_info="palf_id:1007") [2024-06-25 17:00:19.326163] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AC6-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b5b74f6a0, extra_info="palf_id:1007") [2024-06-25 17:00:19.326170] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AC6-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=848, this=0x2b3b5b74f8d0, extra_info="palf_id:1007") [2024-06-25 17:00:19.326174] INFO [CLOG] handle_submit_task_ (ob_log_replay_service.cpp:1208) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=9] succ to submit log task to replay service(to_submit_lsn={lsn:5896353983088}, to_submit_scn={val:1719306019044617002, v:0}, replay_status={ls_id_:{id:1007}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306019326161, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:237788, ret_code_:0}}, next_to_submit_lsn_:{lsn:5896353983154}, next_to_submit_scn_:{val:1719306019044617003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713332543410073001, v:0}, iterator_:{iterator_impl:{buf_:0x2b4f5c205000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:5896353983032}, end_lsn:{lsn:5896353983154}, read_buf:{buf_len_:3682304, buf_:0x2b4f5c205000}, block_size:67104768, log_storage_:0x2b3b5b755470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306019044617002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306019044617002, v:0}, data_checksum:2442502576, flag:0}}, init_mode_version:0, accumulate_checksum:287639555, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306019044617002, v:0}}}}}) [2024-06-25 17:00:19.327756] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABF4533-0006166171EF4868-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301551, send_timestamp=1719305997026205) [2024-06-25 17:00:19.329656] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [62719][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=20] get_max_applied_scn(scn={val:1, v:0}, this={ls_id_:{id:1}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1, v:0}}) [2024-06-25 17:00:19.329868] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.329898] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.329902] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.329905] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.329907] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.330006] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1005, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.330417] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95434][T1004_L0_G1][T1004][YB420ABA45AD-0006164650E9E66C-0-0] [lt=3] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1039, child={server:"10.186.69.173:2882", region:"sys_region", register_time_us:1719306019329304, keepalive_ts:-1}) [2024-06-25 17:00:19.330426] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95434][T1004_L0_G1][T1004][YB420ABA45AD-0006164650E9E66C-0-0] [lt=9] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1039, self="10.191.69.51:2882", child={server:"10.186.69.173:2882", region:"sys_region", register_time_us:1719306019329304, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:19.331295] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=15] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=0) [2024-06-25 17:00:19.332631] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.337489] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=56000, remain_us=25389687, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=56, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.338198] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=10][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.340415] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95434][T1004_L0_G1][T1004][YB420ABF4538-000616694DAF5331-0-0] [lt=6] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1005, child={server:"10.191.69.56:2882", region:"sys_region", register_time_us:1719306019338542, keepalive_ts:-1}) [2024-06-25 17:00:19.340421] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95434][T1004_L0_G1][T1004][YB420ABF4538-000616694DAF5331-0-0] [lt=6] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1005, self="10.191.69.51:2882", child={server:"10.191.69.56:2882", region:"sys_region", register_time_us:1719306019338542, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:19.343260] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [100423][T1003_FreInfoRe][T1003][Y0-0000000000000000-0-0] [lt=9] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-06-25 17:00:19.343539] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63467][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7156, mem_used=12478464) [2024-06-25 17:00:19.347557] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABF4533-0006166172BEBE90-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301548, send_timestamp=1719305997046008) [2024-06-25 17:00:19.347652] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=80000, remain_us=22477327, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=80, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.350034] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=7] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:19.353912] INFO [STORAGE.TRANS] print_statistics (ob_xa_define.cpp:467) [100461][T1003_ObXAHbWor][T1003][Y0-0000000000000000-0-0] [lt=11] xa statistics(total_active_xa_ctx_count=0, total_standby_clearup_count=0, total_success_xa_start=0, total_failure_xa_start=0, total_success_xa_prepare=0, total_failure_xa_prepare=0, total_success_xa_1pc_commit=0, total_failure_xa_1pc_commit=0, total_success_xa_2pc_commit=0, total_failure_xa_2pc_commit=0, total_failure_xa_rollback=0, total_success_dblink_promotion=0, total_failure_dblink_promotion=0, total_success_dblink=0, total_failure_dblink=0) [2024-06-25 17:00:19.360168] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-0006164325C300DB-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301544, send_timestamp=1719305997058624) [2024-06-25 17:00:19.360209] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.360219] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.360222] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78df9940, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019305674, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.360242] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=20][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78df9940, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019305674, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.360255] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.360260] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.360264] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.360266] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.360269] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.360272] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.360276] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=213, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.360282] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=2248513, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=213, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.360462] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABF4538-00061669548FCBFF-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301542, send_timestamp=1719305997058919) [2024-06-25 17:00:19.362213] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B3D-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf09d470, extra_info="") [2024-06-25 17:00:19.362252] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62460][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.362318] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=78, this=0x2b3aaf0a73b0, extra_info="palf_id:1009") [2024-06-25 17:00:19.362338] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=6, this=0x2b3aaf0a75e0, extra_info="palf_id:1009") [2024-06-25 17:00:19.362462] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=14372361, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=134, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.363070] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62458][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.363080] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F001-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf09d6a0, extra_info="palf_id:1009") [2024-06-25 17:00:19.363086] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F001-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=845, this=0x2b3aaf09d8d0, extra_info="palf_id:1009") [2024-06-25 17:00:19.363110] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62455][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.363272] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62459][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.363457] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62456][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.363648] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62454][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.363712] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62457][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.364714] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [62725][T1_LSArchiveMgr][T1][YB420ABF4533-000616423A1E6725-0-0] [lt=9] gc stale ls task succ [2024-06-25 17:00:19.364724] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1611166, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=217, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.366504] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.366511] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.366513] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.366515] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.366518] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.366723] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC060-000616432547F002-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301541, send_timestamp=1719305997065181) [2024-06-25 17:00:19.367005] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62465][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367028] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62445][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367031] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62467][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367181] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62471][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367247] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62440][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367527] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62473][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367541] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62463][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367582] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62466][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367583] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62461][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367597] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62468][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367608] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62464][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367630] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62435][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367660] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62447][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367732] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62441][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367754] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62469][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367757] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62446][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367768] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62470][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367831] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62442][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367843] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62434][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367860] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62444][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367864] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62443][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368123] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62452][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368125] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62462][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368184] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62449][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368184] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62448][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368184] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62437][BatchIO][T0][Y0-0000000000000000-0-0] [lt=9] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368192] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62451][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368227] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62438][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368228] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62436][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368245] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62439][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368311] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62450][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.373727] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62475][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.374573] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=14] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289107427328, global_cache_size=552067284992, tenant_max_wash_size=83976305, tenant_min_wash_size=83976305, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:19.375727] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3329254, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=206, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.379449] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100271][T1004_L0_G0][T1004][YB420ABF4539-00061646327C8726-0-0] [lt=11][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.380445] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [62735][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[{id:1}]) [2024-06-25 17:00:19.383444] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100059][T1004_L0_G0][T1004][YB420ABF4539-00061646327C8726-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.385339] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.387744] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABF4533-0006166172BEBE95-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301555, send_timestamp=1719305997086188) [2024-06-25 17:00:19.393924] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.394927] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [62932][T1_L0_G0][T1][YB420ABA45AB-000616C80E7FB3EF-0-0] [lt=12] [RS_MGR] force set rs list(master_rs="10.186.69.171:2882") [2024-06-25 17:00:19.394939] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [62932][T1_L0_G0][T1][YB420ABA45AB-000616C80E7FB3EF-0-0] [lt=8] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"10.191.69.51:2882", server_id:1, rs_addr:"10.186.69.171:2882", rs_server_status:1, epoch_id:1}, hb_response={zone:"zone1", server:"10.191.69.51:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1713323604978888, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=13) [2024-06-25 17:00:19.396758] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.399061] INFO [STORAGE.TRANS] print_statistics (ob_xa_define.cpp:467) [99389][T1004_ObXAHbWor][T1004][Y0-0000000000000000-0-0] [lt=12] xa statistics(total_active_xa_ctx_count=0, total_standby_clearup_count=0, total_success_xa_start=0, total_failure_xa_start=0, total_success_xa_prepare=0, total_failure_xa_prepare=0, total_success_xa_1pc_commit=0, total_failure_xa_1pc_commit=0, total_success_xa_2pc_commit=0, total_failure_xa_2pc_commit=0, total_failure_xa_rollback=0, total_success_dblink_promotion=0, total_failure_dblink_promotion=0, total_success_dblink=0, total_failure_dblink=0) [2024-06-25 17:00:19.400512] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.402338] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.402358] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e68a080, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221528085, assoc_session_id:3221528085, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306019132035, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2bb3c49201f8, id:3221528085, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e68a080}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:19.406115] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.415919] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.415980] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=5] ====== check clog disk timer task ====== [2024-06-25 17:00:19.415991] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=7] get_disk_usage(ret=0, capacity(MB):=5529, used(MB):=4371) [2024-06-25 17:00:19.416006] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=6] cannot_recycle_log_size statistics(cannot_recycle_log_size=288379996, threshold=1739461755, need_update_checkpoint_scn=false) [2024-06-25 17:00:19.417282] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=8][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.425652] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:19.426220] INFO [LIB] stat (utility.h:1197) [99517][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=13] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=8, this=0x2b3ae6e6c8f8, extra_info="") [2024-06-25 17:00:19.426230] INFO [LIB] stat (utility.h:1197) [99517][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=1, this=0x2b3ae6e6cb28, extra_info="") [2024-06-25 17:00:19.427880] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [100428][T1003_ReqMemEvi][T1003][Y0-0000000000000000-0-0] [lt=7] run rewrite rule refresh task(rule_mgr_->tenant_id_=1003) [2024-06-25 17:00:19.428223] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100574][T1003_L0_G0][T1003][YB420ABA45B0-00061644F03C36DA-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1008}}) [2024-06-25 17:00:19.428245] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100574][T1003_L0_G0][T1004][YB420ABA45B0-00061644F03C36DA-0-0] [lt=14] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306019138440002, v:0}, arg={tenant_id:1004, ls_id:{id:1008}}, result={tenant_id:1004, ls_id:{id:1008}, cur_readable_scn:{val:1719306019138440002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:19.429927] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.429959] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.429965] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.429967] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.429970] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.432686] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.433416] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF59D3-0-0] [lt=4] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=1333) [2024-06-25 17:00:19.433796] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.436766] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1719306019200138, ctx_timeout_ts=-1, worker_timeout_ts=1719306019200138, default_timeout=1000000) [2024-06-25 17:00:19.436779] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=13][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.436785] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] batch renew ls locations failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.436792] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] renew location failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.436798] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] fail to get location(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.436801] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail to get log stream location leader(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.436809] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4012] get leader failed(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.436814] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.436819] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] fetch new range failed(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.436831] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=1][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:35, local_retry_times:35, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:19.436839] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-4012] failed to close result(close_ret=-4383, ret=-4012) [2024-06-25 17:00:19.436846] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:19.436853] INFO [SERVER] process_final (ob_inner_sql_connection.cpp:734) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6] slow inner sql(last_ret=-4012, sql={ObIExecutor:, sql:"SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')"}, process_time=10236341) [2024-06-25 17:00:19.436858] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')"}, aret=-4012, ret=-4012) [2024-06-25 17:00:19.436862] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1002, sql=SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')) [2024-06-25 17:00:19.436917] WDIAG [SERVER] test (ob_query_retry_ctrl.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-5066] execution was terminated(ret=-5066) [2024-06-25 17:00:19.436922] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:false, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:19.436931] WDIAG [SERVER] response_result (ob_sync_plan_driver.cpp:120) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-5066] close result set fail(cret=-4012) [2024-06-25 17:00:19.436941] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:366) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4] sending error packet(ob_error=-5066, client error=-5066, extra_err_info=NULL, lbt()="0x12a0688c 0xa2161e0 0xa1c51c6 0x4e7e290 0x4e35890 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x2b3993985ea5 0x2b3993c9896d") [2024-06-25 17:00:19.436970] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:549) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6] dump txn free route audit_record(value=5, session->get_sessid()=3221518213, session->get_proxy_sessid()=774413746465638104) [2024-06-25 17:00:19.436977] WDIAG [SERVER] do_process (obmp_query.cpp:823) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute query fail(ret=-5066, timeout_timestamp=1719306019200140) [2024-06-25 17:00:19.436993] WDIAG [SQL.SESSION] set_session_state_ (ob_basic_session_info.cpp:5794) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] session is killed(ret=-5066, sessid_=3221518213, proxy_sessid_=774413746465638104, state=1) [2024-06-25 17:00:19.437004] WDIAG [RPC.OBMYSQL] async_write_data (ob_sql_sock_session.cpp:163) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4009] sock has error(ret=-4009) [2024-06-25 17:00:19.437009] WDIAG [SERVER] flush_buffer (obmp_packet_sender.cpp:1032) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4009] write response fail(ret=-4009) [2024-06-25 17:00:19.437011] WDIAG [SERVER] process (obmp_query.cpp:402) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to flush_buffer(tmp_ret=-4009) [2024-06-25 17:00:19.438267] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=14][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.438793] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABA45AB-000616C80A1FC7F5-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1007}}) [2024-06-25 17:00:19.438808] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABA45AB-000616C80A1FC7F5-0-0] [lt=9] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306019044617003, v:0}, arg={tenant_id:1004, ls_id:{id:1007}}, result={tenant_id:1004, ls_id:{id:1007}, cur_readable_scn:{val:1719306019044617003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:19.446580] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.447300] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=1] will sleep(sleep_us=57000, remain_us=25279876, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=57, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.448180] INFO pktc_resp_cb_on_msg (pktc_resp.h:70) [62234][pnio1][T0][Y0-0000000000000000-0-0] [lt=8] PNIO resp cb not found: packet_id=1035875784 [2024-06-25 17:00:19.449952] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=31240684, this={dio_aligned_buf:{buf_write_offset:614, buf_padding_size:3482, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:50864145, truncate_used_ts:14707412}, log_block_size:67108864, dir_fd:819, io_fd:813}, aligned_buf_len=4096, aligned_buf="ÿÿ", aligned_block_offset=31240192, buf_len=122, total_write_size_=12060, total_write_size_after_dio_=417792, ob_pwrite_used_ts=7556, count=99, each_pwrite_cost=76) [2024-06-25 17:00:19.450170] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F004-0-0] [lt=2] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=96, avg_log_submit_to_slide_time=287) [2024-06-25 17:00:19.450177] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1342, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.454902] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABE88-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b6d5470, extra_info="") [2024-06-25 17:00:19.454987] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=73, this=0x2b3b5b6df3b0, extra_info="palf_id:1008") [2024-06-25 17:00:19.454995] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:19.455014] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3b5b6df5e0, extra_info="palf_id:1008") [2024-06-25 17:00:19.455789] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ACF-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5b6d56a0, extra_info="palf_id:1008") [2024-06-25 17:00:19.455794] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ACF-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=848, this=0x2b3b5b6d58d0, extra_info="palf_id:1008") [2024-06-25 17:00:19.460343] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:213, local_retry_times:213, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.460353] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.460356] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.460358] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.460364] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.460521] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABF4538-00061669548FCC03-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301550, send_timestamp=1719305997158970) [2024-06-25 17:00:19.461232] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=8] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-06-25 17:00:19.461257] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=10] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_refresh=false, gts_local_cache={srr:[mts=1719306019132963], gts:1719306019133168000, latest_srr:[mts=1719306019461252]}) [2024-06-25 17:00:19.461586] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62239][pnio1][T0][YB420ABF4533-0006166171AF65D9-0-0] [lt=9][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:19.461595] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62239][pnio1][T0][YB420ABF4533-0006166171AF65D9-0-0] [lt=9] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:19.461606] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62239][pnio1][T0][YB420ABF4533-0006166171AF65D9-0-0] [lt=5] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306019461247]}) [2024-06-25 17:00:19.464305] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=19348804, this={dio_aligned_buf:{buf_write_offset:3509, buf_padding_size:587, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:51289944, truncate_used_ts:14807760}, log_block_size:67108864, dir_fd:825, io_fd:838}, aligned_buf_len=4096, aligned_buf=",ÖoìÜ3¶_æˆ", aligned_block_offset=19345408, buf_len=113, total_write_size_=10840, total_write_size_after_dio_=372736, ob_pwrite_used_ts=6508, count=89, each_pwrite_cost=73) [2024-06-25 17:00:19.466559] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.466567] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.466569] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.466572] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.466574] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.466653] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:19.466790] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-000616432547F006-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305997165246) [2024-06-25 17:00:19.467499] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.470935] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9478332, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=166, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.472297] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=5173044, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=194, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.477526] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62237][pnio1][T1001][YB420ABF4533-00061661718F6536-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.481501] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=1] will sleep(sleep_us=81000, remain_us=22343477, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=81, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.485403] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.486572] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [100471][T1003_LSArchive][T1003][YB420ABF4533-0006164249CE6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:19.493485] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [100481][T1003_HAService][T1003][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:19.497960] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F4-0-0] [lt=4] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11031, add_timestamp:1719306019497953}) [2024-06-25 17:00:19.497969] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF03-0-0] [lt=25] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11031, vtable_type={tenant_id:1, type:4}) [2024-06-25 17:00:19.498800] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF03-0-0] [lt=8] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11031, locations=[{table_id:11031, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:19.498833] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF03-0-0] [lt=32] success to process renew task(task={tenant_id:1, table_id:11031, add_timestamp:1719306019497953}, locations=[{table_id:11031, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:19.499688] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [99409][T1004_HAService][T1004][Y0-0000000000000000-0-0] [lt=10] start do ls ha handler(ls_id_array_=[{id:1006}, {id:1005}, {id:1007}, {id:1008}, {id:1039}]) [2024-06-25 17:00:19.505237] INFO [LIB] stat (utility.h:1197) [62744][T1_ReplaySrv1][T1][Y0-0000000000000000-0-0] [lt=3] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=9, this=0x2b3a3f4048f8, extra_info="") [2024-06-25 17:00:19.505246] INFO [LIB] stat (utility.h:1197) [62744][T1_ReplaySrv1][T1][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=40, this=0x2b3a3f404b28, extra_info="") [2024-06-25 17:00:19.514193] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.514202] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.514213] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2094582, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=214, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.516357] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=14218467, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=135, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.518368] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301556, send_timestamp=1719305997216811) [2024-06-25 17:00:19.518492] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1457398, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=218, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.519141] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.521975] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.525283] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=5] ====== check clog disk timer task ====== [2024-06-25 17:00:19.525291] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=6] get_disk_usage(ret=0, capacity(MB):=230400, used(MB):=0) [2024-06-25 17:00:19.525297] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=4] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=72477573120, need_update_checkpoint_scn=false) [2024-06-25 17:00:19.525746] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.529500] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3175482, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=207, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.529985] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.530017] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=17, this=0x2b3b5a817470, extra_info="") [2024-06-25 17:00:19.530028] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5a884e50, extra_info="id:1005") [2024-06-25 17:00:19.530038] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.530041] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.530043] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.530046] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.530098] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=75, this=0x2b3b5a8213b0, extra_info="palf_id:1005") [2024-06-25 17:00:19.530116] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8215e0, extra_info="palf_id:1005") [2024-06-25 17:00:19.530221] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164325962AD1-0-0] [lt=1][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301545, send_timestamp=1719305997228675) [2024-06-25 17:00:19.530266] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325962AD2-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305997228722) [2024-06-25 17:00:19.530307] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AD2-0-0] [lt=3] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8176a0, extra_info="palf_id:1005") [2024-06-25 17:00:19.530314] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AD2-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=298, this=0x2b3b5a8178d0, extra_info="palf_id:1005") [2024-06-25 17:00:19.531518] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.532740] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.533271] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.534638] INFO [COMMON] replace_fragment_node (ob_kvcache_map.cpp:696) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] Cache replace map node details(ret=0, replace_node_count=0, replace_time=0, replace_start_pos=59900000, replace_num=100000) [2024-06-25 17:00:19.534649] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=10] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=10) [2024-06-25 17:00:19.535901] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [100464][T1003_ReqMemEvi][T1003][Y0-0000000000000000-0-0] [lt=11] sql audit evict task end(request_manager_->get_tenant_id()=1003, evict_high_mem_level=1932735283, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=82982, mem_used=195495936) [2024-06-25 17:00:19.537123] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC061-00061643952B83BC-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=50620681, send_timestamp=1719305968916442) [2024-06-25 17:00:19.538308] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=4][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.541271] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.546788] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F5-0-0] [lt=9][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=1326, pos=316, bitmap=NULL) [2024-06-25 17:00:19.546967] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F5-0-0] [lt=10][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=412, pos=303, bitmap=NULL) [2024-06-25 17:00:19.547168] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F5-0-0] [lt=5][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=76, len=384, pos=368, bitmap=NULL) [2024-06-25 17:00:19.547377] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F5-0-0] [lt=4][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=446, pos=293, bitmap=NULL) [2024-06-25 17:00:19.547573] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F5-0-0] [lt=5][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=421, pos=314, bitmap=NULL) [2024-06-25 17:00:19.550143] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:19.554958] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABE8C-0-0] [lt=3] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1342, avg_group_log_size=122) [2024-06-25 17:00:19.556443] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.558068] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=1] will sleep(sleep_us=58000, remain_us=25169107, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=58, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.559032] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.560268] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-0006164325C300E3-0-0] [lt=5][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301548, send_timestamp=1719305997258719) [2024-06-25 17:00:19.560293] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C300E3-0-0] [lt=11] [PALF STAT GROUP LOG INFO](palf_id=1010, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:19.560374] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1010, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:19.560593] INFO [LIB] stat (utility.h:1197) [63266][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=4] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a872048f8, extra_info="") [2024-06-25 17:00:19.560602] INFO [LIB] stat (utility.h:1197) [63266][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=8] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=1, this=0x2b3a87204b28, extra_info="") [2024-06-25 17:00:19.561295] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.566582] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.566588] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.566591] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.566593] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.566596] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.567102] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.574785] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=10] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289111621632, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:19.576896] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.577627] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62236][pnio1][T1001][YB420ABF4533-00061661718F6537-0-0] [lt=10][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.585457] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.589884] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=11] Cache wash clean map node details(ret=0, clean_node_count=6, clean_time=0, clean_start_pos=172738137, clean_num=402653) [2024-06-25 17:00:19.591853] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F6-0-0] [lt=4][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=73, len=343, pos=314, bitmap=NULL) [2024-06-25 17:00:19.592917] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.594678] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.601499] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF_DUMP](palf_id=1006, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1006, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:96260143, max_lsn:75469139831, max_scn:{val:1719306019461421000, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:75469139831}, reserved_buffer_size:41943040, available_buffer_size:33554432, readable_begin_lsn:{lsn:75427196791}}, last_submit_lsn:{lsn:75469139709}, last_submit_end_lsn:{lsn:75469139831}, last_submit_log_id:96260143, last_submit_log_pid:1, max_flushed_lsn:{lsn:75469139709}, max_flushed_end_lsn:{lsn:75469139831}, max_flushed_log_pid:1, committed_end_lsn:{lsn:75469139831}, last_slide_log_id:96260143, last_slide_scn:{val:1719306019461421000, v:0}, last_slide_lsn:{lsn:75469139709}, last_slide_end_lsn:{lsn:75469139831}, last_slide_log_pid:1, last_slide_log_accum_checksum:3592405039, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2b3a9537a530}, [StateMgr]={this:0x2b3a9537e2b0, self:"10.191.69.51:2882", palf_id:1006, role:"LEADER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, leader:"10.191.69.51:2882", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:96260144, is_changing_config_with_arb:false, reconfirm_start_time_us:1713323710371480, palf_role_change_cb:0x2b3a95384b70, allow_vote:true}, [ConfigMgr]={palf_id:1006, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:1, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"STARTWORKING"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:1}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:1}, persistent_config_version:{proposal_id:1, config_seq:2}, ms_ack_list:0, resend_config_version:{proposal_id:1, config_seq:2}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:-1, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:-1, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2b3a9537bc70}, [ModeMgr]={palf_id:1006, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697225787001, v:0}}, accepted_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697225787001, v:0}}, last_submit_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697225787001, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:1713323710571689, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1006, is_inited:true, min_block_max_scn:{val:1718653213037023001, v:0}, min_block_id:1112, min_block_min_scn:{val:1718597848550157004, v:0}, base_lsn_for_block_gc:{lsn:75425759232}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, log_config_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, log_snapshot_meta:{version:1, base_lsn:{lsn:75425759232}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697225787001, v:0}}}, log_meta_storage:{log_tail:{lsn:3170304}, readable_log_tail:{lsn:3170304}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:75492864000}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1006}, block_mgr:{log_dir:"/data/clog/tenant_1002/1006/meta", dir_fd:815, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:63934464, block_header_serialize_buf_:0x2b3a95380438, flashback_version:0}, log_storage:{log_tail:{lsn:75469139831}, readable_log_tail:{lsn:75469139831}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:75425759232}, min_scn:{val:1719269782386373001, v:0}, curr_block_id:1124, palf_id:1006}, block_mgr:{log_dir:"/data/clog/tenant_1002/1006/log", dir_fd:816, min_block_id:1112, max_block_id:1125, curr_writable_block_id:1124}, logical_block_size_:67104768, curr_block_writable_size_:23724169, block_header_serialize_buf_:0x2b3a953820b8, flashback_version:0}, palf_epoch:1, last_purge_throttling_ts:1713323710370969, this:0x2b3a9537e9f0}, [Reconfirm]={palf_id:1006, self:"10.191.69.51:2882", state:"FINISHED", new_proposal_id:1, prepare_log_ack_list:3"10.191.69.56:2882";"10.189.192.96:2882";"10.186.69.171:2882";, curr_paxos_follower_list:4{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, majority_cnt:3, majority_max_log_server:"10.191.69.56:2882", majority_max_accept_pid:0, majority_max_lsn:{lsn:0}, saved_end_lsn:{lsn:0}, last_submit_prepare_req_time_us:1713323710371613, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2b3a9537e430}) [2024-06-25 17:00:19.601616] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=114] [PALF_DUMP](palf_id=1006, self="10.191.69.51:2882", ack_info_list=[{member:{server:"10.186.69.171:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019467396, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.186.69.176:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019467403, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.189.192.96:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466808, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.191.69.51:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466671, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.191.69.56:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466731, last_flushed_end_lsn:{lsn:75469139831}}]) [2024-06-25 17:00:19.601629] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=11] current log_task status(palf_id=1006, self="10.191.69.51:2882", log_id=96260144, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[{member:{server:"10.186.69.171:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019467396, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.186.69.176:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019467403, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.189.192.96:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466808, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.191.69.51:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466671, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.191.69.56:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466731, last_flushed_end_lsn:{lsn:75469139831}}]) [2024-06-25 17:00:19.603478] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [62703][T1_LSMetaCh][T1][YB420ABF4533-0006166172AE6CE2-0-0] [lt=47] fetch root ls success(local_cluster_id=1713323324, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.171:2882", sql_port:2881, role:1, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607828518, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.176:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:4, zone:"zone4", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607829360, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.189.192.96:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:5, zone:"zone5", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607830236, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.51:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607831113, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.56:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["10.186.69.171:2882", "10.191.69.51:2882", "10.191.69.56:2882", "10.186.69.176:2882", "10.189.192.96:2882"]) [2024-06-25 17:00:19.616322] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=1] will sleep(sleep_us=82000, remain_us=22208656, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=82, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.617904] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63299][T1002_LSArchive][T1002][YB420ABF4533-000616423EEE6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:19.618223] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63484][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=8] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:19.624596] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9324671, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=167, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.624613] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=22] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:19.626081] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5019259, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=195, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.626529] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.628554] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.630032] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.630060] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.630065] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.630067] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.630070] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.630275] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABDC060-0006164325962AD6-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301552, send_timestamp=1719305997328723) [2024-06-25 17:00:19.630279] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-0006164325962AD7-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301548, send_timestamp=1719305997328730) [2024-06-25 17:00:19.630328] INFO [LIB] stat (utility.h:1197) [99510][T1004_ApplySrv4][T1004][YB420ABF4533-0006166111AEB555-0-0] [lt=2] [APPLY STAT CB APPEND COST TIME](cur_stat_count=10, stat_interval=5000000, avg (count/cost)=22, this=0x2b3b5a82a280, extra_info="") [2024-06-25 17:00:19.630337] INFO [LIB] stat (utility.h:1197) [99510][T1004_ApplySrv4][T1004][YB420ABF4533-0006166111AEB555-0-0] [lt=9] [APPLY STAT CB IN QUEUE TIME](cur_stat_count=10, stat_interval=5000000, avg (count/cost)=263, this=0x2b3b5a82a4b0, extra_info="") [2024-06-25 17:00:19.630342] INFO [LIB] stat (utility.h:1197) [99510][T1004_ApplySrv4][T1004][YB420ABF4533-0006166111AEB555-0-0] [lt=5] [APPLY STAT CB WAIT COMMIT TIME](cur_stat_count=10, stat_interval=5000000, avg (count/cost)=0, this=0x2b3b5a82a6e0, extra_info="") [2024-06-25 17:00:19.630345] INFO [LIB] stat (utility.h:1197) [99510][T1004_ApplySrv4][T1004][YB420ABF4533-0006166111AEB555-0-0] [lt=3] [APPLY STAT CB EXECUTE TIME](cur_stat_count=10, stat_interval=5000000, avg (count/cost)=-503, this=0x2b3b5a82a910, extra_info="") [2024-06-25 17:00:19.630350] INFO [LIB] stat (utility.h:1197) [99510][T1004_ApplySrv4][T1004][YB420ABF4533-0006166111AEB555-0-0] [lt=4] [APPLY STAT CB TOTAL TIME](cur_stat_count=10, stat_interval=5000000, avg (count/cost)=-217, this=0x2b3b5a82ab40, extra_info="") [2024-06-25 17:00:19.632762] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.638362] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:120) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6] tenant info is invalid(ret=-4076, query_ts=0, standby_scn={val:0, v:0}, this={inited:true, last_id:-1, tenant_id:1004, epoch:-1, self:"10.191.69.51:2882", switch_to_leader_ts:-1}) [2024-06-25 17:00:19.638375] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:134) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=13] ObStandbyTimestampService state(*this={inited:true, last_id:-1, tenant_id:1004, epoch:-1, self:"10.191.69.51:2882", switch_to_leader_ts:-1}) [2024-06-25 17:00:19.638381] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.651736] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [100456][T1003_LCLSender][T1003][Y0-0000000000000000-0-0] [lt=16] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=30358025, total_destructed_detector=30358025, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x2b3b43a382b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-06-25 17:00:19.658637] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.660332] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164325C300E7-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301565, send_timestamp=1719305997358767) [2024-06-25 17:00:19.660359] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C300E7-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1010, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.660640] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABF4538-00061669548FCC0A-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301542, send_timestamp=1719305997359097) [2024-06-25 17:00:19.663083] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.666663] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=19, this=0x2b3a9537b470, extra_info="") [2024-06-25 17:00:19.666674] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=9, avg_log_batch_cnt=1, total_group_log_size=1098, avg_group_log_size=122) [2024-06-25 17:00:19.666681] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT APPEND DATA SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1098) [2024-06-25 17:00:19.666687] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT APPEND COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a9534ee50, extra_info="id:1006") [2024-06-25 17:00:19.666691] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.666695] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.666697] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.666699] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.666703] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.666757] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=79, this=0x2b3a953853b0, extra_info="palf_id:1006") [2024-06-25 17:00:19.666774] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=1098) [2024-06-25 17:00:19.666784] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=4, this=0x2b3a953855e0, extra_info="palf_id:1006") [2024-06-25 17:00:19.666908] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-000616432547F00C-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305997365364) [2024-06-25 17:00:19.666953] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F00C-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a9537b6a0, extra_info="palf_id:1006") [2024-06-25 17:00:19.666961] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F00C-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=303, this=0x2b3a9537b8d0, extra_info="palf_id:1006") [2024-06-25 17:00:19.666964] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F00C-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=24, avg_log_submit_to_flush_time=96, avg_log_submit_to_slide_time=278) [2024-06-25 17:00:19.668087] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1940707, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=215, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.669969] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=59000, remain_us=25057206, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=59, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.670178] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=14064645, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=136, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.672154] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1303737, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=219, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.677655] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62232][pnio1][T1001][YB420ABF4533-00061661718F6538-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.683073] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3021908, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=208, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.685524] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.685711] INFO [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:262) [63507][T1002_ArcSender][T1002][YB420ABF4533-00061642411E6725-0-0] [lt=8] ObArchiveSender is running(thread_index=2) [2024-06-25 17:00:19.694252] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.702358] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABF4533-000616604F4F58D5-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620689, send_timestamp=1719305969081668) [2024-06-25 17:00:19.702549] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABF4533-000616604F4F58D5-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620676, send_timestamp=1719305969081872) [2024-06-25 17:00:19.704348] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [100252][T1004_L0_G0][T1004][YB420ABF4533-000616611FFEABF8-0-0] [lt=10] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1004, table_id:11021, add_timestamp:1719306019704341}) [2024-06-25 17:00:19.704355] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF04-0-0] [lt=25] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1004, table_id=11021, vtable_type={tenant_id:1004, type:3}) [2024-06-25 17:00:19.704473] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC088-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a5d1db470, extra_info="") [2024-06-25 17:00:19.704566] INFO [LIB] stat (utility.h:1197) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=12] [PALF STAT WRITE LOG COST TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=76, this=0x2b3a5d1e53b0, extra_info="palf_id:1") [2024-06-25 17:00:19.704579] INFO [LIB] stat (utility.h:1214) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=10] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=6, this=0x2b3a40ffddb0, extra_info="") [2024-06-25 17:00:19.704592] INFO [LIB] stat (utility.h:1197) [62591][T1_LogIOCb0][T1][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=10, this=0x2b3a5d1e55e0, extra_info="palf_id:1") [2024-06-25 17:00:19.705174] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF04-0-0] [lt=7] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1004, table_id=11021, locations=[{table_id:11021, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:19.705200] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF04-0-0] [lt=26] success to process renew task(task={tenant_id:1004, table_id:11021, add_timestamp:1719306019704341}, locations=[{table_id:11021, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:19.706850] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABF4533-000616611FFEABF8-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301559, send_timestamp=1719305997405291) [2024-06-25 17:00:19.707026] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100316][T1004_L0_G0][T1004][YB420ABF4533-000616611FFEABF8-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.707048] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABF4533-000616611FFEABF8-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301544, send_timestamp=1719305997405504) [2024-06-25 17:00:19.707199] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABF4533-000616611FFEABF8-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620679, send_timestamp=1719305969086519) [2024-06-25 17:00:19.708503] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [62728][T1_ArcSender][T1][YB420ABF4533-000616423A4E6725-0-0] [lt=10][errcode=-4018] try free send task failed(ret=-4018) [2024-06-25 17:00:19.711325] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99943][T1004_L0_G0][T1004][YB420ABF4533-000616611FFEABF8-0-0] [lt=11][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.711678] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABF4533-000616611FFEABF8-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620678, send_timestamp=1719305969090999) [2024-06-25 17:00:19.726343] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ADA-0-0] [lt=6] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.726603] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F00D-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.726899] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100157][T1004_L0_G0][T1004][YB420ABA45B0-000616443951B6D2-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.730089] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=22, this=0x2bcde0525470, extra_info="") [2024-06-25 17:00:19.730105] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b589fc96e50, extra_info="id:1039") [2024-06-25 17:00:19.730110] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.730140] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.730145] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.730147] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.730150] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.730191] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=36040775, this={dio_aligned_buf:{buf_write_offset:193, buf_padding_size:3903, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:10637072, truncate_used_ts:2870794}, log_block_size:67108864, dir_fd:983, io_fd:846}, aligned_buf_len=4096, aligned_buf="", aligned_block_offset=36040704, buf_len=122, total_write_size_=12294, total_write_size_after_dio_=425984, ob_pwrite_used_ts=6971, count=101, each_pwrite_cost=69) [2024-06-25 17:00:19.730207] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=15] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=71, this=0x2bcde052f3b0, extra_info="palf_id:1039") [2024-06-25 17:00:19.730215] INFO [LIB] stat (utility.h:1214) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbff3b0, extra_info="") [2024-06-25 17:00:19.730223] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=4, this=0x2bcde052f5e0, extra_info="palf_id:1039") [2024-06-25 17:00:19.730364] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ADB-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2bcde05256a0, extra_info="palf_id:1039") [2024-06-25 17:00:19.730370] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ADB-0-0] [lt=6] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=310, this=0x2bcde05258d0, extra_info="palf_id:1039") [2024-06-25 17:00:19.730373] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABDC060-0006164325962ADC-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301548, send_timestamp=1719305997428824) [2024-06-25 17:00:19.731670] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100202][T1004_L0_G0][T1004][YB420ABA45B0-000616443951B6D2-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.732816] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.732969] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=11][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4013, dropped:279, tid:12536}]) [2024-06-25 17:00:19.734715] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=4] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=9) [2024-06-25 17:00:19.735782] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [63302][T1002_ArcSender][T1002][YB420ABF4533-000616423F1E6725-0-0] [lt=8][errcode=-4018] try free send task failed(ret=-4018) [2024-06-25 17:00:19.738416] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=4][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.741950] INFO pn_ratelimit (group.c:551) [62497][IngressService][T0][Y0-0000000000000000-0-0] [lt=6] PNIO set ratelimit as 9223372036854775807 bytes/s, grp_id=2 [2024-06-25 17:00:19.747890] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=13][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:19.747899] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:19.750336] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62748][T1_ReplaySrv5][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:19.752157] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=83000, remain_us=22072821, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=83, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.757471] INFO [STORAGE.TRANS] post_msg (ob_trans_rpc.cpp:525) [62160][ServerGTimer][T1004][YB420ABF4533-000616616D6E83BA-0-0] [lt=9] post txn free route msg(ret=0, msg={type:81, request_id:741221, req_sess_id:3221632376, tx_sess_id:3221932298, tx_id:{txid:2228768677}, sender:"10.191.69.51:2882", receiver:"10.191.69.56:2882"}) [2024-06-25 17:00:19.757662] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [62160][ServerGTimer][T1004][Y0-0000000000000000-0-0] [lt=14] get_disk_usage(ret=0, capacity(MB):=2073600, used(MB):=1658818) [2024-06-25 17:00:19.760372] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325C300EB-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301544, send_timestamp=1719305997458827) [2024-06-25 17:00:19.760670] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABF4538-00061669548FCC0D-0-0] [lt=14][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301547, send_timestamp=1719305997459122) [2024-06-25 17:00:19.766726] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.766733] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.766735] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.766737] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.766740] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.770350] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [99194][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=8] ====== tenant freeze timer task ====== [2024-06-25 17:00:19.771398] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.771409] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:19.771415] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.771422] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.771428] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.773477] INFO [CLOG.EXTLOG] run1 (ob_cdc_service.cpp:172) [99269][T1004_CdcSrv][T1004][Y0-0000000000000000-0-0] [lt=6] query dest_info_ succ(dest_info=[], archive_dest_changed=false) [2024-06-25 17:00:19.773488] INFO [CLOG.EXTLOG] resize_log_ext_handler_ (ob_cdc_service.cpp:632) [99269][T1004_CdcSrv][T1004][Y0-0000000000000000-0-0] [lt=9] finish to resize log external storage handler(current_ts=1719306019773487, tenant_max_cpu=100, valid_ls_count=0, other_ls_count=0, new_concurrency=0) [2024-06-25 17:00:19.774268] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.774277] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.774283] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:19.774289] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.774294] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.777641] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62235][pnio1][T1001][YB420ABF4533-00061661718F6539-0-0] [lt=9][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:19.778114] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.778123] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.778128] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:19.778135] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.778139] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.778355] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9170912, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=168, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.779746] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=4865594, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=196, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.782865] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=60000, remain_us=24944310, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=60, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.783998] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.784008] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.784013] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:19.784019] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.784024] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.785578] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.788498] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.788508] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.788513] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.788520] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.788525] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.788537] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e69ab20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221632491, assoc_session_id:3221632491, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306019518122, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3a97f681f8, id:3221632491, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e69ab20}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:19.790097] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=8] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289111621632, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:19.793675] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.793682] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.793686] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.793692] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.793696] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.799883] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=23] get_max_applied_scn(scn={val:1713323697230007001, v:0}, this={ls_id_:{id:1009}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713323697230007001, v:0}}) [2024-06-25 17:00:19.799897] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=11] get_min_unreplayed_log_info(lsn={lsn:86651600553}, scn={val:1719306019638250001, v:0}, this={ls_id_:{id:1009}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306019763275, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:86651600553}, next_to_submit_scn_:{val:1719306019638250001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713323697230007001, v:0}, iterator_:{iterator_impl:{buf_:0x2b3ac8f21000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:86651600431}, end_lsn:{lsn:86651600553}, read_buf:{buf_len_:90112, buf_:0x2b3ac8f21000}, block_size:67104768, log_storage_:0x2b3aaf0a3470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306019638250000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306019638250000, v:0}, data_checksum:1616820247, flag:1}}, init_mode_version:0, accumulate_checksum:1174171203, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306019638250000, v:0}}}}}) [2024-06-25 17:00:19.808896] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [62724][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=6] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-06-25 17:00:19.808906] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:99) [62724][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=6] [emptytablet] succeed to change tablet to empty shell(ret=0, ret="OB_SUCCESS", ls_cnt=1, times=4154) [2024-06-25 17:00:19.811462] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.811470] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.811474] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.811480] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.811484] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.821319] INFO do_work (ob_rl_mgr.cpp:709) [62262][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=8] swc wakeup.(stat_period_=1000000, ready=false) [2024-06-25 17:00:19.821993] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1786802, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=216, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.822729] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [62723][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=5] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-06-25 17:00:19.822741] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [62723][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=9] [tabletchange] task check ls(ls->get_ls_id()={id:1}, tablet_persist_trigger=0) [2024-06-25 17:00:19.822748] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:228) [62723][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=5] [tabletchange] succeed to gc_tablet(ret=0, ret="OB_SUCCESS", ls_cnt=1, times=4081) [2024-06-25 17:00:19.824048] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=13910775, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=137, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.824064] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.824073] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.824079] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.824085] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.824091] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.825950] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1149940, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=220, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.829933] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [62719][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=11] get_min_unreplayed_log_info(lsn={lsn:35325765024}, scn={val:1719306019747657001, v:0}, this={ls_id_:{id:1}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306019750266, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:35325765024}, next_to_submit_scn_:{val:1719306019747657001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1, v:0}, iterator_:{iterator_impl:{buf_:0x2b3a6c849000, next_round_pread_size:3678208, curr_read_pos:539, curr_read_buf_start_pos:0, curr_read_buf_end_pos:539, log_storage_:{IteratorStorage:{start_lsn:{lsn:35325764485}, end_lsn:{lsn:35325765024}, read_buf:{buf_len_:708608, buf_:0x2b3a6c849000}, block_size:67104768, log_storage_:0x2b3a5d1e1470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306019747657000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:451, scn_:{val:1719306019747657000, v:0}, data_checksum:2836757861, flag:0}}, init_mode_version:0, accumulate_checksum:3568790969, curr_entry_is_padding:0, padding_entry_size:483, padding_entry_scn:{val:1719306019747657000, v:0}}}}}) [2024-06-25 17:00:19.830144] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.830176] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.830181] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.830183] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.830186] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.830223] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=12] [PALF STAT COMMITTED LOG SIZE](palf_id=1039, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.830241] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABF4533-0006166171EF486D-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301546, send_timestamp=1719305997528694) [2024-06-25 17:00:19.830388] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164325962AE1-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301542, send_timestamp=1719305997528845) [2024-06-25 17:00:19.830389] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164325962AE0-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301550, send_timestamp=1719305997528839) [2024-06-25 17:00:19.830440] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AE0-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1039, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=28, avg_log_submit_to_flush_time=90, avg_log_submit_to_slide_time=284) [2024-06-25 17:00:19.830478] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AE1-0-0] [lt=10] [PALF STAT LOG TASK TIME](palf_id=1005, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=20, avg_log_submit_to_flush_time=90, avg_log_submit_to_slide_time=277) [2024-06-25 17:00:19.832870] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.833066] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4012, dropped:25, tid:62821}]) [2024-06-25 17:00:19.833436] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF59E3-0-0] [lt=5] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:19.836658] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:920) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=8] can close safely, do destroy(*s={this:0x2b7098196d70, session_id:3221518213, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:false, reader:{fd:927}, err:5, last_decode_time:1719306009199675, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:686, pending_flag:1, may_handling_flag:false, handler_close_flag:false}) [2024-06-25 17:00:19.836681] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=17] unlock session of tenant(conn.sessid_=3221518213, proxy_sessid=774413746465638104, conn.tenant_id_=1) [2024-06-25 17:00:19.836696] WDIAG [COMMON] get (ob_link_hashmap.h:313) [62929][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=3][errcode=0] uref leak check(node->uref_=1077129732, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x2b3993985ea5 0x2b3993c9896d") [2024-06-25 17:00:19.836689] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=7] connection close(sessid=3221518213, proxy_sessid=774413746465638104, tenant_id=1, server_id=1, from_proxy=true, from_java_client=false, c/s protocol="OB_2_0_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB420ABF4533-0006166133BFE257-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:4, cur_pkt_pos_:6, pkt_rec[0]:{send:obp_mysql_header_{len_:6342, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{send:obp20_header_:{payload_len:14, pkt_seq:0, req_id:2398505, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:539, seq_:2}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[2]:{send:obp20_header_:{payload_len:43, pkt_seq:0, req_id:2398506, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:401, seq_:2}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{receive:obp20_header_:{payload_len:248, pkt_seq:0, req_id:2398507, flag_.flags:62, ext_flags:0}, obp_mysql_header_{len_:244, rec_:248, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[4]:{send:obp_compess_header_{com_len:135, com_seq:1}, obp20_header_:{payload_len:107, pkt_seq:1, req_id:2398507, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:32, seq_:2}, pkt_name:"PKT_ERR", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_compess_header_{com_len:135, com_seq:1}, obp20_header_:{payload_len:107, pkt_seq:1, req_id:2398507, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:75, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}], disconnect_state=4) [2024-06-25 17:00:19.836728] INFO [SERVER] run (obmp_disconnect.cpp:82) [62929][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=7] free session successfully(sessid=3221518213) [2024-06-25 17:00:19.836734] INFO [SERVER] run (obmp_disconnect.cpp:89) [62929][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=6] mark session id unused(sessid=3221518213) [2024-06-25 17:00:19.836840] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=2868141, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=209, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.838470] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.845227] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.845240] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=13][errcode=-4013] [2024-06-25 17:00:19.845245] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.845251] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.845255] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.847812] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-00061643246151E9-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5407, fly_ts=22301547, send_timestamp=1719305997546265) [2024-06-25 17:00:19.849991] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:19.850303] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABF4533-0006166172BEBE9A-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301552, send_timestamp=1719305997548751) [2024-06-25 17:00:19.855991] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AE3-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=89, avg_log_submit_to_slide_time=850) [2024-06-25 17:00:19.856000] INFO [PALF] read (log_cache.cpp:100) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=33] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.860420] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325C300EF-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301545, send_timestamp=1719305997558875) [2024-06-25 17:00:19.860712] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABF4538-00061669548FCC11-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301547, send_timestamp=1719305997559165) [2024-06-25 17:00:19.860733] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABF4538-00061669548FCC11-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1010, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=91, avg_log_submit_to_slide_time=287) [2024-06-25 17:00:19.860739] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT HOT CACHE HIT RATE](palf_id=1010, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.862496] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B4D-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1009, self="10.191.69.51:2882", committed size=967) [2024-06-25 17:00:19.862573] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=967) [2024-06-25 17:00:19.866654] INFO [CLOG] runTimerTask (ob_log_replay_service.cpp:159) [100441][T1003_ReplayPro][T1003][Y0-0000000000000000-0-0] [lt=10] dump tenant replay process(tenant_id=1003, unreplayed_log_size(MB)=0, estimate_time(second)=0, replayed_log_size(MB)=0, last_replayed_log_size(MB)=0, round_cost_time(second)=10, pending_replay_log_size(MB)=0) [2024-06-25 17:00:19.866775] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.866782] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.866784] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.866786] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.866790] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.874966] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164324713061-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=22301542, send_timestamp=1719305997573424) [2024-06-25 17:00:19.877673] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62245][pnio1][T1001][YB420ABF4533-00061661718F653A-0-0] [lt=8][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:19.878396] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:19.878407] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:19.878413] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999946, timeout_timestamp=1719306049877938) [2024-06-25 17:00:19.878420] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.878422] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:19.878427] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:19.878432] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:19.878435] WDIAG [SQL] open (ob_result_set.cpp:157) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:19.878438] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:19.878442] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.878447] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.878454] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=6][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:19.878461] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:19.878465] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:19.878467] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:19.878473] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:19.878479] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4013, ret=-4013) [2024-06-25 17:00:19.878484] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.878489] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:19.878492] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=2][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:19.878495] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] query failed(ret=-4013, conn=0x2b3a78d44050, start=1719306019877938, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.878502] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=6][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:19.878505] WDIAG [SHARE.PT] get_by_tenant (ob_persistent_ls_table.cpp:639) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=2][errcode=-4013] execute sql failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.878530] WDIAG [SHARE.PT] get_by_tenant (ob_ls_table_operator.cpp:252) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] get all ls info by persistent_ls_ failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002) [2024-06-25 17:00:19.878535] WDIAG [SHARE] inner_open_ (ob_ls_table_iterator.cpp:104) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] fail to get ls infos by tenant(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, inner_table_only=false) [2024-06-25 17:00:19.878539] WDIAG [SHARE] next (ob_ls_table_iterator.cpp:71) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] fail to open iterator(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.878543] WDIAG [SERVER] build_replica_map_ (ob_tenant_meta_checker.cpp:334) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] ls table iterator next failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.878547] WDIAG [SERVER] check_ls_table_ (ob_tenant_meta_checker.cpp:214) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] build replica map from ls table failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", mode=0) [2024-06-25 17:00:19.878553] WDIAG [SERVER] check_ls_table (ob_tenant_meta_checker.cpp:188) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] check ls table failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", mode=0) [2024-06-25 17:00:19.878556] WDIAG [SERVER] runTimerTask (ob_tenant_meta_checker.cpp:44) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] fail to check ls meta table(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.880360] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABF4533-0006166172BEBE9F-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301550, send_timestamp=1719305997578809) [2024-06-25 17:00:19.885624] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.889030] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=84000, remain_us=21935949, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=84, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.896753] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=61000, remain_us=24830423, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=61, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.897090] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99938][T1004_L0_G0][T1004][YB420ABA45B1-0006164629D475BC-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.899929] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-06-25 17:00:19.899936] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=6] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-06-25 17:00:19.899995] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.900004] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.900009] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.900016] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.900021] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.901674] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=16] re_register_parent reason(palf_id=1008, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306015899493, last_first_register_time_us=1713323715481338, register_time_us=1719306015899493, curr_time_us=1719306019901673) [2024-06-25 17:00:19.901699] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=11] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882", reason=1, leader="10.186.69.171:2882", register_time_us=1719306019901686, last_submit_register_req_time_us=1719306019901686, register_parent_reason=1) [2024-06-25 17:00:19.901708] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=9] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882") [2024-06-25 17:00:19.902187] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99932][T1004_L0_G0][T1004][YB420ABA45B1-0006164629D475BC-0-0] [lt=8][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.902874] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.902882] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.902886] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:19.902892] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.902896] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.904596] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC08C-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=3427, avg_group_log_size=311) [2024-06-25 17:00:19.904684] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=5] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=3427) [2024-06-25 17:00:19.905427] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A089-0-0] [lt=10] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a5d1db6a0, extra_info="palf_id:1") [2024-06-25 17:00:19.905434] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A089-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=833, this=0x2b3a5d1db8d0, extra_info="palf_id:1") [2024-06-25 17:00:19.906847] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.906855] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.906860] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:19.906864] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.906870] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.911075] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.911084] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.911089] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:19.911096] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.911100] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.920724] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.920734] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.920739] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.920746] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.920750] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.922717] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [62718][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=11] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=969588866, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=408846, mem_used=767425536) [2024-06-25 17:00:19.930201] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.930238] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1005, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:19.930247] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.930251] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.930253] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.930258] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.930445] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-0006164325962AE5-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301549, send_timestamp=1719305997628895) [2024-06-25 17:00:19.932021] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9017246, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=169, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.932926] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.933395] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=4711946, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=197, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.933624] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.933633] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.933639] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.933645] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.933650] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.934780] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=8) [2024-06-25 17:00:19.936491] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.936499] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.936503] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.936509] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.936513] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.938524] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.938591] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.938602] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:19.938608] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.938615] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.938619] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.938808] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [99277][T1004_LockWaitM][T1004][Y0-0000000000000000-0-0] [lt=8] report RowHolderMapper summary info(count=4, bkt_cnt=248) [2024-06-25 17:00:19.944366] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.944375] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.944380] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.944386] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.944390] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.944809] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [42764][T1002_ComTimerT][T1002][Y0-0000000000000000-0-0] [lt=8] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-06-25 17:00:19.950358] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=7] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:19.954139] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.954148] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.954153] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.954159] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.954163] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.960484] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325C300F3-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301546, send_timestamp=1719305997658937) [2024-06-25 17:00:19.960508] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C300F3-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf16f470, extra_info="") [2024-06-25 17:00:19.960602] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=77, this=0x2b3aaf1793b0, extra_info="palf_id:1010") [2024-06-25 17:00:19.960611] INFO [LIB] stat (utility.h:1214) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=5, this=0x2b3a8ce1d3b0, extra_info="") [2024-06-25 17:00:19.960621] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3aaf1795e0, extra_info="palf_id:1010") [2024-06-25 17:00:19.960805] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC8FB-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf16f6a0, extra_info="palf_id:1010") [2024-06-25 17:00:19.960810] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC8FB-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=288, this=0x2b3aaf16f8d0, extra_info="palf_id:1010") [2024-06-25 17:00:19.961748] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=8] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_refresh=false, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306019961741]}) [2024-06-25 17:00:19.961931] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=11] gts statistics(tenant_id=1004, gts_rpc_cnt=10, get_gts_cache_cnt=454, get_gts_with_stc_cnt=0, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-06-25 17:00:19.962050] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62240][pnio1][T0][YB420ABF4533-0006166171AF65DE-0-0] [lt=7][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:19.962060] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62240][pnio1][T0][YB420ABF4533-0006166171AF65DE-0-0] [lt=10] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:19.962074] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62240][pnio1][T0][YB420ABF4533-0006166171AF65DE-0-0] [lt=7] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306019961741]}) [2024-06-25 17:00:19.966837] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.966844] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.966847] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.966849] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.966852] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.967057] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-000616432547F018-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305997665513) [2024-06-25 17:00:19.969608] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [99254][T1004_ReqMemEvi][T1004][Y0-0000000000000000-0-0] [lt=10] run rewrite rule refresh task(rule_mgr_->tenant_id_=1004) [2024-06-25 17:00:19.971943] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.971952] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.971957] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.971963] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.971967] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.975878] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1632916, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=217, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.977776] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62239][pnio1][T1001][YB420ABF4533-00061661718F653B-0-0] [lt=9][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:19.977903] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=13756920, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=138, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.979714] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=996176, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=221, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.981218] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=14] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1004, server_version={version:{val:1719306019370955004, v:0}, total_part_count:5, valid_inner_part_count:0, valid_user_part_count:5}, server_version_delta=610261, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="10.191.69.52:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=58348883, succ_cluster_heartbeat_count=58348883, cluster_heartbeat_interval=50000, local_cluster_version={val:1719305752101344002, v:0}, local_cluster_delta=267879872, force_self_check=false, weak_read_refresh_interval=100000) [2024-06-25 17:00:19.985680] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.987741] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=3] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1) [2024-06-25 17:00:19.987753] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=10] start lib cache evict(tenant_id=1, mem_hold=312475648, mem_limit=2019976805, cache_obj_num=914, cache_node_num=913) [2024-06-25 17:00:19.987760] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=6] end lib cache evict(tenant_id=1, cache_evict_num=0, mem_hold=312475648, mem_limit=2019976805, cache_obj_num=914, cache_node_num=913) [2024-06-25 17:00:19.987765] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=4] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:19.990321] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=12] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289109524480, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:19.990477] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2714504, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=210, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.990038] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=5] Dumping All Cache Objs(alloc_obj_list.count()=914, alloc_obj_list=[{obj_id:1, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:2, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:97, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176332}, {obj_id:98, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276716}, {obj_id:99, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:112, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:737569, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219837}, {obj_id:274, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303715}, {obj_id:275, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:248080}, {obj_id:276, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271604}, {obj_id:277, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216033}, {obj_id:278, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195958}, {obj_id:279, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183659}, {obj_id:280, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176078}, {obj_id:281, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196297}, {obj_id:1180050, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209023}, {obj_id:282, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:283, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:284, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179794}, {obj_id:285, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230228}, {obj_id:286, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188271}, {obj_id:287, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196930}, {obj_id:288, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174576}, {obj_id:289, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301933}, {obj_id:290, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211414}, {obj_id:291, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224502}, {obj_id:292, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190726}, {obj_id:293, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202255}, {obj_id:294, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199095}, {obj_id:295, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199093}, {obj_id:296, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193076}, {obj_id:297, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214118}, {obj_id:298, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199546}, {obj_id:299, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199766}, {obj_id:301, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190029}, {obj_id:303, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186863}, {obj_id:304, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185594}, {obj_id:306, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:307, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200193}, {obj_id:308, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193157}, {obj_id:309, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188399}, {obj_id:310, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185761}, {obj_id:311, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:312, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185793}, {obj_id:313, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:314, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185738}, {obj_id:315, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192013}, {obj_id:316, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187261}, {obj_id:317, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183532}, {obj_id:318, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210345}, {obj_id:319, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183340}, {obj_id:320, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188170}, {obj_id:321, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183730}, {obj_id:322, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191888}, {obj_id:323, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183436}, {obj_id:324, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184586}, {obj_id:325, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188962}, {obj_id:326, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:308297}, {obj_id:327, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:251495}, {obj_id:329, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282230}, {obj_id:330, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225422}, {obj_id:389, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183962}, {obj_id:390, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182906}, {obj_id:391, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:392, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:394, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:395, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:396, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:398, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:399, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:400, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:402, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:404, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178712}, {obj_id:482, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214699}, {obj_id:483, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193516}, {obj_id:484, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200871}, {obj_id:485, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201183}, {obj_id:486, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200215}, {obj_id:487, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202195}, {obj_id:490, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191892}, {obj_id:491, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:492, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:493, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:494, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:495, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:496, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:498, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:501, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179663}, {obj_id:503, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185730}, {obj_id:504, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:505, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157686}, {obj_id:510, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:515, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157686}, {obj_id:518, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157704}, {obj_id:519, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:521, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178056}, {obj_id:522, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176744}, {obj_id:523, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174432}, {obj_id:524, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214695}, {obj_id:525, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178793}, {obj_id:526, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:528, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207447}, {obj_id:529, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:275361}, {obj_id:530, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179806}, {obj_id:531, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179440}, {obj_id:532, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191447}, {obj_id:533, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:275362}, {obj_id:534, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218570}, {obj_id:535, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214216}, {obj_id:536, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208800}, {obj_id:543, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276362}, {obj_id:544, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181654}, {obj_id:545, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192407}, {obj_id:546, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:547, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:307440}, {obj_id:548, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:549, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:550, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:551, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:552, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276714}, {obj_id:553, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181974}, {obj_id:554, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192767}, {obj_id:555, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:556, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:557, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:558, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:559, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:560, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:562, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:565, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205804}, {obj_id:566, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197099}, {obj_id:567, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191367}, {obj_id:568, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177328}, {obj_id:569, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182496}, {obj_id:570, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:571, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:572, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180277}, {obj_id:574, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:575, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181673}, {obj_id:577, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196012}, {obj_id:578, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186809}, {obj_id:580, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191323}, {obj_id:581, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:582, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:586, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:589, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214431}, {obj_id:595, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:607, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:608, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201420}, {obj_id:609, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:610, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:611, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:612, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188872}, {obj_id:613, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:299219}, {obj_id:614, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182108}, {obj_id:616, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179156}, {obj_id:617, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173136}, {obj_id:618, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184752}, {obj_id:619, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186441}, {obj_id:620, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179449}, {obj_id:621, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178725}, {obj_id:622, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223877}, {obj_id:623, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:624, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:626, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:627, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197179}, {obj_id:630, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184682}, {obj_id:631, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183630}, {obj_id:638, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196470}, {obj_id:641, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:642, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282175}, {obj_id:643, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192056}, {obj_id:644, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186353}, {obj_id:645, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180529}, {obj_id:646, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173811}, {obj_id:647, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172864}, {obj_id:658, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157686}, {obj_id:659, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157711}, {obj_id:668, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:669, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:672, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:673, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:680, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:696, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199763}, {obj_id:697, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199763}, {obj_id:698, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179616}, {obj_id:699, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179616}, {obj_id:700, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192472}, {obj_id:701, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173811}, {obj_id:703, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199763}, {obj_id:704, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199763}, {obj_id:705, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199763}, {obj_id:706, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174869}, {obj_id:714, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:715, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:716, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:166010}, {obj_id:717, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:162314}, {obj_id:718, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157829}, {obj_id:719, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:724, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178368}, {obj_id:725, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:159523}, {obj_id:727, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194538}, {obj_id:730, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176677}, {obj_id:731, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:305791}, {obj_id:732, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185321}, {obj_id:733, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174560}, {obj_id:734, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172752}, {obj_id:737, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:739, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196384}, {obj_id:740, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173852}, {obj_id:744, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182369}, {obj_id:752, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175760}, {obj_id:753, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180460}, {obj_id:755, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176751}, {obj_id:771, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185223}, {obj_id:772, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184511}, {obj_id:781, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:786, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210446}, {obj_id:787, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:825, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1180787, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207498}, {obj_id:1180962, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303379}, {obj_id:1454, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193581}, {obj_id:50733, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193543}, {obj_id:2399, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221425}, {obj_id:2423, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2447, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2528, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181621}, {obj_id:1182546, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221553}, {obj_id:641850, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:333444}, {obj_id:1182580, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1182617, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:258305}, {obj_id:1182618, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179635}, {obj_id:789398, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205871}, {obj_id:1182707, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193403}, {obj_id:150468, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196073}, {obj_id:1183028, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:262574}, {obj_id:397215, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193944}, {obj_id:839774, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175536}, {obj_id:1086494, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220700}, {obj_id:693718, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:306743}, {obj_id:841223, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218155}, {obj_id:251559, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:573738}, {obj_id:251564, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:995399}, {obj_id:251572, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186890}, {obj_id:251575, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229551}, {obj_id:251578, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292553}, {obj_id:251595, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1136868, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189832}, {obj_id:1136869, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189565}, {obj_id:1136870, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177216}, {obj_id:399522, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:353488}, {obj_id:1137043, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188840}, {obj_id:1087936, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245307}, {obj_id:1087937, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:248426}, {obj_id:1087944, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:335259}, {obj_id:1087945, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:385283}, {obj_id:1087948, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:398171}, {obj_id:1087949, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:481681}, {obj_id:1087950, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:413928}, {obj_id:1087951, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:505577}, {obj_id:1087952, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:402727}, {obj_id:1087953, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:490052}, {obj_id:1087954, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:373499}, {obj_id:1087955, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:444164}, {obj_id:1087958, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:355472}, {obj_id:1087959, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:418640}, {obj_id:793660, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:330128}, {obj_id:793662, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276666}, {obj_id:155139, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:548658, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197785}, {obj_id:893070, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219019}, {obj_id:745601, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:326781}, {obj_id:696557, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199066}, {obj_id:8498, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8499, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:500677}, {obj_id:8500, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190714}, {obj_id:8501, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223442}, {obj_id:8510, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176371}, {obj_id:8514, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177545}, {obj_id:8515, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172928}, {obj_id:8517, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204160}, {obj_id:8518, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188778}, {obj_id:8519, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184970}, {obj_id:8521, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:8522, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185752}, {obj_id:8525, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174043}, {obj_id:8527, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174080}, {obj_id:8529, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175223}, {obj_id:8530, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191631}, {obj_id:8531, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:8532, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177270}, {obj_id:8533, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188473}, {obj_id:8534, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188530}, {obj_id:8536, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175596}, {obj_id:8537, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176756}, {obj_id:8538, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179720}, {obj_id:8539, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173811}, {obj_id:8540, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179286}, {obj_id:8541, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184717}, {obj_id:8542, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183168}, {obj_id:8543, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191567}, {obj_id:8544, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203564}, {obj_id:8545, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213399}, {obj_id:8550, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179898}, {obj_id:8551, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:8552, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:8553, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:8554, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172528}, {obj_id:8556, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:8559, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172160}, {obj_id:8569, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181013}, {obj_id:8570, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178561}, {obj_id:8571, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179597}, {obj_id:8572, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181930}, {obj_id:8573, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191549}, {obj_id:8574, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185675}, {obj_id:8575, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271426}, {obj_id:8577, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:255998}, {obj_id:8578, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232003}, {obj_id:8579, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301272}, {obj_id:8580, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234280}, {obj_id:8582, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202310}, {obj_id:8590, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182942}, {obj_id:8592, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202971}, {obj_id:8593, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182392}, {obj_id:8594, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218911}, {obj_id:8595, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182448}, {obj_id:8598, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204268}, {obj_id:8599, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179164}, {obj_id:8608, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194888}, {obj_id:8613, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191819}, {obj_id:696958, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192822}, {obj_id:8790, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174539}, {obj_id:1188723, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:296704}, {obj_id:1188736, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183934}, {obj_id:1188737, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:265701}, {obj_id:1188738, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:305908}, {obj_id:1188739, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256316}, {obj_id:1188740, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:299497}, {obj_id:1139773, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231817}, {obj_id:1139774, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227447}, {obj_id:1139775, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234072}, {obj_id:1139776, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231307}, {obj_id:1139777, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240818}, {obj_id:1139778, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242250}, {obj_id:1139779, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:236311}, {obj_id:1139780, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235035}, {obj_id:1139781, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:243082}, {obj_id:1139782, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247885}, {obj_id:1139783, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238568}, {obj_id:1139784, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238553}, {obj_id:1139786, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:310506}, {obj_id:1139787, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:347917}, {obj_id:1139788, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:387000}, {obj_id:1139789, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:465236}, {obj_id:1139790, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:440971}, {obj_id:1139791, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:545609}, {obj_id:1139792, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:454412}, {obj_id:1139793, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:563727}, {obj_id:1139794, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292497}, {obj_id:1139795, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:323145}, {obj_id:1139796, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:416211}, {obj_id:1139797, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:509604}, {obj_id:9298, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175422}, {obj_id:107718, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200349}, {obj_id:9516, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174576}, {obj_id:1140302, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:285958}, {obj_id:1140303, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320103}, {obj_id:993262, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179166}, {obj_id:993368, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179331}, {obj_id:993390, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179169}, {obj_id:993440, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179298}, {obj_id:551520, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188170}, {obj_id:551523, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:568093}, {obj_id:551525, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187522}, {obj_id:551529, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196997}, {obj_id:551533, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1027006}, {obj_id:551546, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:393312}, {obj_id:551558, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:650014, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192296}, {obj_id:1191636, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:263303}, {obj_id:1191637, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276482}, {obj_id:1191638, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:368939}, {obj_id:1191639, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:436931}, {obj_id:1191640, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:371228}, {obj_id:1191641, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:442099}, {obj_id:1191642, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:377976}, {obj_id:1191643, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:450423}, {obj_id:1191644, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:328494}, {obj_id:1191645, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:374846}, {obj_id:1191646, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:330766}, {obj_id:1191647, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:381562}, {obj_id:1191648, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297098}, {obj_id:1191649, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:330767}, {obj_id:1191650, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279006}, {obj_id:1191651, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:300721}, {obj_id:1191652, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:309107}, {obj_id:1191653, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:319929}, {obj_id:1191654, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:249785}, {obj_id:1191655, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:254408}, {obj_id:1191656, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205023}, {obj_id:1142646, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:293344}, {obj_id:651095, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:269588}, {obj_id:651096, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232922}, {obj_id:995247, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286678}, {obj_id:995248, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198552}, {obj_id:995249, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199145}, {obj_id:995250, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279990}, {obj_id:995251, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179330}, {obj_id:995271, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180091}, {obj_id:995272, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181318}, {obj_id:946365, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207965}, {obj_id:897362, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184764}, {obj_id:996721, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217721}, {obj_id:849254, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:323413}, {obj_id:407282, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:570916}, {obj_id:407286, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1001046}, {obj_id:407289, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1166431}, {obj_id:407294, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1032702}, {obj_id:407298, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:258169}, {obj_id:407299, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191773}, {obj_id:407308, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:112756, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260624}, {obj_id:112757, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:264190}, {obj_id:112758, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177088}, {obj_id:112759, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181113}, {obj_id:112760, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179110}, {obj_id:112761, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176843}, {obj_id:112766, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242573}, {obj_id:112767, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192979}, {obj_id:112768, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192408}, {obj_id:112769, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:284099}, {obj_id:112770, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174589}, {obj_id:112771, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256377}, {obj_id:112772, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176247}, {obj_id:112773, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:269247}, {obj_id:112774, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241866}, {obj_id:112775, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195217}, {obj_id:112776, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:112777, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:112782, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:938545}, {obj_id:112787, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:269947}, {obj_id:112788, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1828995}, {obj_id:112789, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:632776}, {obj_id:112793, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197485}, {obj_id:112794, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:505900}, {obj_id:112795, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:306517}, {obj_id:112796, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209204}, {obj_id:112800, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199028}, {obj_i [2024-06-25 17:00:19.990576] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=537] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:20.005757] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.005766] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.005770] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:20.005776] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.005780] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.011648] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=62000, remain_us=24715528, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=62, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:20.021246] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [63452][T1001_LockWaitM][T1001][Y0-0000000000000000-0-0] [lt=9] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-06-25 17:00:20.022383] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [62691][T1_ReplayProces][T1][Y0-0000000000000000-0-0] [lt=7] get_replay_process success(id={id:1}, replayed_log_size=35325765685, unreplayed_log_size=0) [2024-06-25 17:00:20.022404] INFO [CLOG] runTimerTask (ob_log_replay_service.cpp:159) [62691][T1_ReplayProces][T1][Y0-0000000000000000-0-0] [lt=9] dump tenant replay process(tenant_id=1, unreplayed_log_size(MB)=0, estimate_time(second)=0, replayed_log_size(MB)=33689, last_replayed_log_size(MB)=33689, round_cost_time(second)=10, pending_replay_log_size(MB)=0) [2024-06-25 17:00:20.023060] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [100346][T1003_PalfGC][T1003][Y0-0000000000000000-0-0] [lt=20] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):230400, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):230400, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:1}) [2024-06-25 17:00:20.025738] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=21] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:20.026469] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AE9-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=97, avg_log_submit_to_slide_time=837) [2024-06-25 17:00:20.026475] INFO [PALF] read (log_cache.cpp:100) [99521][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=12] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:20.026853] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=85000, remain_us=21798125, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=85, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:20.027458] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63474][T1001_LSArchive][T1001][YB420ABF4533-000616423F8E6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:20.027720] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99934][T1004_L0_G0][T1004][YB420ABF4538-0006166915BFA5CD-0-0] [lt=8][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.030257] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] [PALF STAT GROUP LOG INFO](palf_id=1039, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:20.030267] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT APPEND DATA SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1220) [2024-06-25 17:00:20.030275] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.030303] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT APPEND DATA SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1220) [2024-06-25 17:00:20.030309] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.030312] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.030315] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.030318] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.030388] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:20.030528] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-0006164325962AEA-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301554, send_timestamp=1719305997728974) [2024-06-25 17:00:20.030536] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325962AEB-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305997728992) [2024-06-25 17:00:20.031906] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100139][T1004_L0_G0][T1004][YB420ABF4538-0006166915BFA5CD-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.032983] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.033522] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1342) [2024-06-25 17:00:20.036496] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.036506] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:20.036511] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:20.036516] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.036523] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.038579] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.040920] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.040929] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.040933] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:20.040940] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.040945] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.050196] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F01A-0-0] [lt=2] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:20.052883] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [99392][T1004_ReqMemEvi][T1004][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1004, evict_high_mem_level=20669530113, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=8345583, mem_used=15148855296) [2024-06-25 17:00:20.053519] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63292][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=9] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7278, mem_used=12478464) [2024-06-25 17:00:20.055320] INFO [LIB] stat (utility.h:1214) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=7, this=0x2b3afcbfec70, extra_info="") [2024-06-25 17:00:20.060536] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325C300F7-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301548, send_timestamp=1719305997758987) [2024-06-25 17:00:20.060877] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABF4533-00061661728FC8FF-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301550, send_timestamp=1719305997759327) [2024-06-25 17:00:20.062619] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B55-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1009, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=8, avg_log_batch_cnt=0, total_group_log_size=967, avg_group_log_size=120) [2024-06-25 17:00:20.063454] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F01B-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1009, self="10.191.69.51:2882", total_slide_log_cnt=8, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=96, avg_log_submit_to_slide_time=839) [2024-06-25 17:00:20.063458] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=14] [PALF STAT HOT CACHE HIT RATE](palf_id=1009, read_size=967, hit_cnt=8, read_cnt=8, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:20.066893] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.066901] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.066904] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.066906] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:20.066909] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:20.071484] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.071493] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.071498] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:20.071504] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.071508] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.075207] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 0.93, rt: 101.00) [2024-06-25 17:00:20.075217] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SYS](tenant_id=500, io_status=sys_group_name: INSPECT_BAD_BLOCK_IO, mode: read, size: 2097152.00, iops: 0.93, rt: 768.00) [2024-06-25 17:00:20.075223] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:20.075290] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=20] [IO STATUS SYS](tenant_id=1004, io_status=sys_group_name: SLOG_IO, mode: write, size: 4915.20, iops: 4.66, rt: 76.00) [2024-06-25 17:00:20.075296] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS CONFIG](tenant_id=1004, ref_cnt=2, io_config={group_num_:0, memory_limit_:724775731200, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:100}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=12763552, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:20.075305] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075312] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075315] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075319] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075323] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075326] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075329] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075334] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075337] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075340] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075343] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075347] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075350] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075353] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075357] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075360] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.077838] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62237][pnio1][T1001][YB420ABF4533-00061661718F653C-0-0] [lt=8][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:20.085736] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.085779] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=8863488, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=170, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:20.087158] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=4558183, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=198, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:20.088684] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [100465][T1003_TenantWea][T1003][Y0-0000000000000000-0-0] [lt=10] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1003, server_version={version:{val:1719306015462183000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0}, server_version_delta=4626500, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="10.191.69.52:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=58369728, succ_cluster_heartbeat_count=58369728, cluster_heartbeat_interval=50000, local_cluster_version={val:1719305992926967000, v:0}, local_cluster_delta=27161716, force_self_check=false, weak_read_refresh_interval=100000) [2024-06-25 17:00:20.100181] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=6] ====== check clog disk timer task ====== [2024-06-25 17:00:20.100195] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=7] get_disk_usage(ret=0, capacity(MB):=2073600, used(MB):=1658818) [2024-06-25 17:00:20.100209] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=6] cannot_recycle_log_size statistics(cannot_recycle_log_size=524426637471, threshold=652298158080, need_update_checkpoint_scn=false) [2024-06-25 17:00:20.108800] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=12] new mysql sessid created(conn.sessid_=3221540615, support_ssl=false) [2024-06-25 17:00:20.108824] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=10] sm conn init succ(conn.sessid_=3221540615, sess.client_addr_="10.191.69.51:48016") [2024-06-25 17:00:20.108833] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=6] accept one succ(*s={this:0x2b3add9f8030, session_id:3221540615, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:20.109047] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x2b3add9f8030, session_id:3221540615, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:20.109057] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=8] kill and revert session(conn.sessid_=3221540615, proxy_sessid=0, server_id=1, ret=0) [2024-06-25 17:00:20.109061] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:918) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=4] sock ref clean, do destroy(*s={this:0x2b3add9f8030, session_id:3221540615, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:20.109069] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:223) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=7] mark session id unused(conn.sessid_=3221540615) [2024-06-25 17:00:20.109072] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=2] connection close(sessid=3221540615, proxy_sessid=0, tenant_id=0, server_id=1, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=false, ret=0, trace_id=Y0-0000000000000000-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:0], disconnect_state=0) [2024-06-25 17:00:20.110228] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=4] re_register_parent reason(palf_id=1008, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306016008017, last_first_register_time_us=1713332561478106, register_time_us=1719306016008017, curr_time_us=1719306020110228) [2024-06-25 17:00:20.110257] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=15] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882", reason=1, leader="10.186.69.176:2882", register_time_us=1719306020110244, last_submit_register_req_time_us=1719306020110244, register_parent_reason=1) [2024-06-25 17:00:20.110265] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=7] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882") [2024-06-25 17:00:20.110270] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=4] re_register_parent reason(palf_id=1007, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306016008049, last_first_register_time_us=1713332560477990, register_time_us=1719306016008049, curr_time_us=1719306020110270) [2024-06-25 17:00:20.110282] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=6] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1007, self="10.191.69.51:2882", reason=1, leader="10.186.69.171:2882", register_time_us=1719306020110277, last_submit_register_req_time_us=1719306020110277, register_parent_reason=1) [2024-06-25 17:00:20.110287] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=5] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1007, self="10.191.69.51:2882") [2024-06-25 17:00:20.119863] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [62918][T1_L0_G0][T1][YB420ABF4533-000616611FBF57A2-0-0] [lt=7] succeed to calculate base table sharding info(table_id=18446744073709551614, ref_table_id=104, is_global_index=false) [2024-06-25 17:00:20.119887] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [62918][T1_L0_G0][T1][YB420ABF4533-000616611FBF57A2-0-0] [lt=12] total rowcount, use statistics(table_meta_info_.table_row_count_=10, table_meta_info_.average_row_size_=2.060000000000000000e+02, table_meta_info_.micro_block_count_=1, table_meta_info_.part_size_=2.060000000000000000e+03) [2024-06-25 17:00:20.124325] WDIAG [STORAGE.TRANS] process_ (ob_gti_rpc.h:178) [62245][pnio1][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=11][errcode=0] gti rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:20.124336] WDIAG [SQL.PC] common_free (ob_lib_cache_object_manager.cpp:141) [62918][T1_L0_G0][T1][YB420ABF4533-000616611FBF57A2-0-0] [lt=11][errcode=0] set logical del time(cache_obj->get_logical_del_time()=6134803531101, cache_obj->added_lc()=false, cache_obj->get_object_id()=1202421, cache_obj->get_tenant_id()=1, lbt()="0x12a0688c 0xaa7c9e4 0x4e44d8a 0x4e36df8 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x2b3993985ea5 0x2b3993c9896d") [2024-06-25 17:00:20.125715] WDIAG get_refactored (ob_hashtable.h:1100) [62774][T1_L0_G0][T1][YB420ABF4533-0006166163BEF587-0-0] [lt=11][errcode=-4201] hashtable not init [2024-06-25 17:00:20.125722] WDIAG [SQL.SESSION] get_cursor (ob_sql_session_info.cpp:1439) [62774][T1_L0_G0][T1][YB420ABF4533-0006166163BEF587-0-0] [lt=7][errcode=-4016] get cursor info failed(cursor_id=1194334, get_sessid()=3221488526) [2024-06-25 17:00:20.125744] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB06-0-0] [lt=10] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:20.125976] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC8C5-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:20.126796] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F01D-0-0] [lt=11] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=90, avg_log_submit_to_slide_time=835) [2024-06-25 17:00:20.127434] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=63000, remain_us=24599741, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=63, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:20.129742] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [100474][T1003_ArcSender][T1003][YB420ABF4533-0006164249FE6725-0-0] [lt=9][errcode=-4018] try free send task failed(ret=-4018) [2024-06-25 17:00:20.129767] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1479027, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=218, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:20.130300] INFO eloop_run (eloop.c:143) [62232][pnio1][T0][Y0-0000000000000000-0-0] [lt=7] PNIO [ratelimit] time: 1719306020130299, bytes: 29137866649075, bw: 0.669858 MB/s, add_ts: 1000513, add_bytes: 702757 [2024-06-25 17:00:20.130313] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.130351] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.130355] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.130358] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.130360] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.130547] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325962AEF-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301546, send_timestamp=1719305997829000) [2024-06-25 17:00:20.131769] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=13603054, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=139, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:20.133038] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.133345] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4383, dropped:947, tid:62272}]) [2024-06-25 17:00:20.133459] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF59EF-0-0] [lt=9] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b61c69470, extra_info="") [2024-06-25 17:00:20.133465] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=1][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.133474] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:20.133478] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb21722e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020079578, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.133502] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=24][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb21722e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020079578, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.133514] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:20.133520] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:20.133525] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:20.133529] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:20.133532] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.133537] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:20.133543] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=222, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.133540] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=67, this=0x2b3b61c733b0, extra_info="palf_id:1006") [2024-06-25 17:00:20.133550] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=6] will sleep(sleep_us=100000, remain_us=842340, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=222, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:20.133566] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=15] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3b61c735e0, extra_info="palf_id:1006") [2024-06-25 17:00:20.133735] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AF1-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b61c696a0, extra_info="palf_id:1006") [2024-06-25 17:00:20.133740] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AF1-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=273, this=0x2b3b61c698d0, extra_info="palf_id:1006") [2024-06-25 17:00:20.134845] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=7) [2024-06-25 17:00:20.138635] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.138774] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=7][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:20.138783] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.138790] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=4][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999943, timeout_timestamp=1719306050138311) [2024-06-25 17:00:20.138797] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:20.138800] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:20.138805] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:20.138808] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=3][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:20.138812] WDIAG [SQL] open (ob_result_set.cpp:157) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=4][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:20.138815] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=3][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:20.138818] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=2][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}) [2024-06-25 17:00:20.138824] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=5][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.138829] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=4][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:20.138838] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=7][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.138840] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=2][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.138843] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:20.138850] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=4][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:20.138856] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}, aret=-4013, ret=-4013) [2024-06-25 17:00:20.138859] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000) [2024-06-25 17:00:20.138865] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:20.138868] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:20.138873] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] query failed(ret=-4013, conn=0x2b3b5a082050, start=1719306020138310, sql=SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000) [2024-06-25 17:00:20.138878] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:20.138882] WDIAG [SERVER.OMT] fetch_all_srs (ob_tenant_srs.cpp:456) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute sql failed(sql=SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000, ret=-4013, tenant_id=1001) [2024-06-25 17:00:20.138908] WDIAG [SERVER.OMT] refresh_srs (ob_tenant_srs.cpp:254) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] failed to fetch ObSrsCacheSnapShot(ret=-4013, is_sys=true) [2024-06-25 17:00:20.138913] WDIAG [SERVER.OMT] runTimerTask (ob_tenant_srs.cpp:387) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] failed to refresh sys srs(ret=-4013, tenant_srs_->remote_sys_srs_version_=0, tenant_srs_->local_sys_srs_version_=0) [2024-06-25 17:00:20.141691] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62420][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142244] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62425][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142460] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62416][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142524] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cp