[MEMORY] hold= 221,184 used= 214,432 count= 1 avg_used= 214,432 block_cnt= 1 chunk_cnt= 1 mod=CompSuggestMgr [MEMORY] hold= 212,992 used= 196,624 count= 2 avg_used= 98,312 block_cnt= 2 chunk_cnt= 1 mod=DRTaskMap [MEMORY] hold= 212,992 used= 196,624 count= 2 avg_used= 98,312 block_cnt= 2 chunk_cnt= 2 mod=DdlQue [MEMORY] hold= 212,992 used= 180,400 count= 4 avg_used= 45,100 block_cnt= 4 chunk_cnt= 4 mod=ProxySchemSQL [MEMORY] hold= 208,000 used= 206,336 count= 26 avg_used= 7,936 block_cnt= 26 chunk_cnt= 20 mod=ObSyncTabletSeq [MEMORY] hold= 196,608 used= 196,224 count= 3 avg_used= 65,408 block_cnt= 3 chunk_cnt= 2 mod=AsyncTaskQueue [MEMORY] hold= 194,480 used= 156,712 count= 11 avg_used= 14,246 block_cnt= 10 chunk_cnt= 7 mod=BucketLock [MEMORY] hold= 180,224 used= 172,064 count= 1 avg_used= 172,064 block_cnt= 1 chunk_cnt= 1 mod=TenantMBList [MEMORY] hold= 180,224 used= 135,784 count= 11 avg_used= 12,344 block_cnt= 11 chunk_cnt= 6 mod=ReferedMap [MEMORY] hold= 163,840 used= 147,792 count= 2 avg_used= 73,896 block_cnt= 2 chunk_cnt= 1 mod=HashNodNexWaiMa [MEMORY] hold= 163,504 used= 155,416 count= 2 avg_used= 77,708 block_cnt= 2 chunk_cnt= 2 mod=OB_DISK_REP [MEMORY] hold= 160,624 used= 159,272 count= 21 avg_used= 7,584 block_cnt= 21 chunk_cnt= 3 mod=TenaSpaTabIdSet [MEMORY] hold= 155,648 used= 147,624 count= 1 avg_used= 147,624 block_cnt= 1 chunk_cnt= 1 mod=UsrRuleMap [MEMORY] hold= 147,456 used= 110,736 count= 6 avg_used= 18,456 block_cnt= 6 chunk_cnt= 6 mod=HashBuckSysConf [MEMORY] hold= 144,880 used= 143,656 count= 19 avg_used= 7,560 block_cnt= 19 chunk_cnt= 4 mod=SysTableNameMap [MEMORY] hold= 138,240 used= 136,320 count= 30 avg_used= 4,544 block_cnt= 30 chunk_cnt= 13 mod=[T]ObTraceEvent [MEMORY] hold= 136,000 used= 134,912 count= 17 avg_used= 7,936 block_cnt= 17 chunk_cnt= 16 mod=SqlSessiVarMap [MEMORY] hold= 122,560 used= 118,904 count= 2 avg_used= 59,452 block_cnt= 2 chunk_cnt= 2 mod=TenaSchForCacMa [MEMORY] hold= 122,560 used= 118,904 count= 2 avg_used= 59,452 block_cnt= 2 chunk_cnt= 2 mod=RefrFullScheMap [MEMORY] hold= 122,560 used= 118,904 count= 2 avg_used= 59,452 block_cnt= 2 chunk_cnt= 2 mod=MemMgrMap [MEMORY] hold= 122,560 used= 118,904 count= 2 avg_used= 59,452 block_cnt= 2 chunk_cnt= 2 mod=MemMgrForLiboMa [MEMORY] hold= 114,688 used= 111,096 count= 1 avg_used= 111,096 block_cnt= 1 chunk_cnt= 1 mod=NonPartTenMap [MEMORY] hold= 114,688 used= 111,096 count= 1 avg_used= 111,096 block_cnt= 1 chunk_cnt= 1 mod=IndNameMap [MEMORY] hold= 114,336 used= 106,088 count= 2 avg_used= 53,044 block_cnt= 2 chunk_cnt= 2 mod=RetryCtrl [MEMORY] hold= 112,000 used= 111,104 count= 14 avg_used= 7,936 block_cnt= 14 chunk_cnt= 12 mod=ObDDLCommitClog [MEMORY] hold= 112,000 used= 111,104 count= 14 avg_used= 7,936 block_cnt= 14 chunk_cnt= 14 mod=ObDDLStartClogC [MEMORY] hold= 112,000 used= 111,104 count= 14 avg_used= 7,936 block_cnt= 14 chunk_cnt= 7 mod=HashMapArray [MEMORY] hold= 110,480 used= 105,224 count= 81 avg_used= 1,299 block_cnt= 41 chunk_cnt= 14 mod=ObGuard [MEMORY] hold= 106,496 used= 98,312 count= 1 avg_used= 98,312 block_cnt= 1 chunk_cnt= 1 mod=TmpFileManager [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 9 mod=PrepareOk [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 9 mod=acceptOK [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 9 mod=AddrList [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 9 mod=followerRenew [MEMORY] hold= 84,992 used= 55,968 count= 394 avg_used= 142 block_cnt= 2 chunk_cnt= 2 mod=TLDecoderCtx [MEMORY] hold= 82,432 used= 40,608 count= 542 avg_used= 74 block_cnt= 272 chunk_cnt= 27 mod=Coro [MEMORY] hold= 81,920 used= 46,960 count= 5 avg_used= 9,392 block_cnt= 5 chunk_cnt= 5 mod=TenCompProgMgr [MEMORY] hold= 80,000 used= 79,360 count= 10 avg_used= 7,936 block_cnt= 10 chunk_cnt= 9 mod=PlanVaIdx [MEMORY] hold= 73,728 used= 69,664 count= 1 avg_used= 69,664 block_cnt= 1 chunk_cnt= 1 mod=SuperBlockBuffe [MEMORY] hold= 73,728 used= 65,600 count= 1 avg_used= 65,600 block_cnt= 1 chunk_cnt= 1 mod=TCREF [MEMORY] hold= 64,816 used= 60,696 count= 4 avg_used= 15,174 block_cnt= 4 chunk_cnt= 3 mod=TmpFileStoreMap [MEMORY] hold= 64,000 used= 63,488 count= 8 avg_used= 7,936 block_cnt= 8 chunk_cnt= 8 mod=InneSqlConnPool [MEMORY] hold= 60,544 used= 51,200 count= 19 avg_used= 2,694 block_cnt= 15 chunk_cnt= 5 mod=TGTimer [MEMORY] hold= 58,736 used= 44,720 count= 215 avg_used= 208 block_cnt= 45 chunk_cnt= 10 mod=Scheduler [MEMORY] hold= 57,344 used= 55,368 count= 1 avg_used= 55,368 block_cnt= 1 chunk_cnt= 1 mod=DepInfoTaskQ [MEMORY] hold= 49,152 used= 32,768 count= 2 avg_used= 16,384 block_cnt= 2 chunk_cnt= 1 mod=CACHE_TNT_LST [MEMORY] hold= 48,832 used= 44,824 count= 2 avg_used= 22,412 block_cnt= 2 chunk_cnt= 2 mod=DDLSpeedCtrl [MEMORY] hold= 48,000 used= 47,616 count= 6 avg_used= 7,936 block_cnt= 6 chunk_cnt= 5 mod=MacroIdList [MEMORY] hold= 41,264 used= 37,264 count= 2 avg_used= 18,632 block_cnt= 2 chunk_cnt= 1 mod=TaskRunnerSer [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucPooUniMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucTenPooMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucSerUniMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucIdConfMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucNamPooMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HasBucSerMigUnM [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucNamConMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HasBucConRefCoM [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=ProxySessBuck [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SessHoldMapBuck [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucConPooMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HashBucIdPoolMa [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=TableConnBucket [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=ObLongopsMgr [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SqlLoadData [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 5 mod=ObTsTenantInfoN [MEMORY] hold= 31,440 used= 7,344 count= 306 avg_used= 24 block_cnt= 153 chunk_cnt= 24 mod=[T]MemoryContex [MEMORY] hold= 30,464 used= 29,976 count= 7 avg_used= 4,282 block_cnt= 7 chunk_cnt= 6 mod=MemstoreAllocat [MEMORY] hold= 24,944 used= 20,784 count= 2 avg_used= 10,392 block_cnt= 2 chunk_cnt= 2 mod=SqlNio [MEMORY] hold= 24,672 used= 24,264 count= 6 avg_used= 4,044 block_cnt= 6 chunk_cnt= 3 mod=ScheObSchemAren [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=FuncRuleMap [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=leakMap [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=GrpIdNameMap [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=ServerCkptSlogH [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=GrpNameIdMap [MEMORY] hold= 24,256 used= 20,152 count= 2 avg_used= 10,076 block_cnt= 2 chunk_cnt= 2 mod=TbltRefreshMap [MEMORY] hold= 24,224 used= 21,672 count= 2 avg_used= 10,836 block_cnt= 2 chunk_cnt= 1 mod=SchemaStatuMap [MEMORY] hold= 24,000 used= 23,808 count= 3 avg_used= 7,936 block_cnt= 3 chunk_cnt= 2 mod=ObBlackListHash [MEMORY] hold= 23,616 used= 23,424 count= 3 avg_used= 7,808 block_cnt= 3 chunk_cnt= 3 mod=ProxySessNode [MEMORY] hold= 23,520 used= 23,160 count= 5 avg_used= 4,632 block_cnt= 5 chunk_cnt= 5 mod=SqlPx [MEMORY] hold= 23,520 used= 23,160 count= 5 avg_used= 4,632 block_cnt= 5 chunk_cnt= 5 mod=RebuildCtx [MEMORY] hold= 21,504 used= 20,352 count= 16 avg_used= 1,272 block_cnt= 16 chunk_cnt= 2 mod=IO_GROUP_MAP [MEMORY] hold= 18,832 used= 14,040 count= 65 avg_used= 216 block_cnt= 39 chunk_cnt= 31 mod=ObFuture [MEMORY] hold= 18,544 used= 17,408 count= 17 avg_used= 1,024 block_cnt= 12 chunk_cnt= 9 mod=timer [MEMORY] hold= 18,416 used= 18,144 count= 4 avg_used= 4,536 block_cnt= 4 chunk_cnt= 2 mod=DeviceMng [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=HasBucTimZonInM [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=StorageHADiag [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=ServerLogPool [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 8,992 count= 1 avg_used= 8,992 block_cnt= 1 chunk_cnt= 1 mod=IO_HEALTH [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=MemDumpMap [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=LinkArray [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=GenSchemVersMap [MEMORY] hold= 16,384 used= 9,336 count= 1 avg_used= 9,336 block_cnt= 1 chunk_cnt= 1 mod=InnerLobHash [MEMORY] hold= 16,384 used= 12,296 count= 1 avg_used= 12,296 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 16,320 used= 16,000 count= 5 avg_used= 3,200 block_cnt= 5 chunk_cnt= 5 mod=TenantTZ [MEMORY] hold= 16,032 used= 15,904 count= 2 avg_used= 7,952 block_cnt= 2 chunk_cnt= 1 mod=UpgProcSet [MEMORY] hold= 16,000 used= 15,872 count= 2 avg_used= 7,936 block_cnt= 2 chunk_cnt= 1 mod=CommSysVarDefVa [MEMORY] hold= 16,000 used= 15,872 count= 2 avg_used= 7,936 block_cnt= 2 chunk_cnt= 2 mod=DisplayList [MEMORY] hold= 15,744 used= 15,616 count= 2 avg_used= 7,808 block_cnt= 2 chunk_cnt= 2 mod=SessHoldMapNode [MEMORY] hold= 14,288 used= 8,352 count= 87 avg_used= 96 block_cnt= 39 chunk_cnt= 7 mod=PThread [MEMORY] hold= 10,240 used= 9,920 count= 5 avg_used= 1,984 block_cnt= 5 chunk_cnt= 5 mod=LobManager [MEMORY] hold= 8,288 used= 8,048 count= 3 avg_used= 2,682 block_cnt= 3 chunk_cnt= 3 mod=TenantInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=SchemaRowKey [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=TbltRefTasks [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DDLClogCtrl [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ObBLValue [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=lock_diagnose [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DutyTask [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=TbltReSrv [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=RetryTask [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ServerBlacklist [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ServerCidMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ServerRegioMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ServerIdcMap [MEMORY] hold= 7,968 used= 7,904 count= 1 avg_used= 7,904 block_cnt= 1 chunk_cnt= 1 mod=LuaHandler [MEMORY] hold= 7,872 used= 7,808 count= 1 avg_used= 7,808 block_cnt= 1 chunk_cnt= 1 mod=HasNodTzInfM [MEMORY] hold= 7,808 used= 6,528 count= 16 avg_used= 408 block_cnt= 16 chunk_cnt= 15 mod=[T]ObTLDecoderC [MEMORY] hold= 7,360 used= 7,000 count= 5 avg_used= 1,400 block_cnt= 5 chunk_cnt= 5 mod=GtiRequestRpc [MEMORY] hold= 7,040 used= 6,889 count= 2 avg_used= 3,444 block_cnt= 2 chunk_cnt= 2 mod=RpcProcessor [MEMORY] hold= 6,960 used= 6,640 count= 5 avg_used= 1,328 block_cnt= 5 chunk_cnt= 5 mod=GtiRpcProxy [MEMORY] hold= 5,376 used= 5,248 count= 2 avg_used= 2,624 block_cnt= 1 chunk_cnt= 1 mod=RootContext [MEMORY] hold= 5,216 used= 4,608 count= 9 avg_used= 512 block_cnt= 8 chunk_cnt= 8 mod=SqlString [MEMORY] hold= 5,184 used= 4,608 count= 9 avg_used= 512 block_cnt= 7 chunk_cnt= 7 mod=SqlSessiQuerSql [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmReq [MEMORY] hold= 4,480 used= 3,816 count= 9 avg_used= 424 block_cnt= 7 chunk_cnt= 7 mod=SequenceMap [MEMORY] hold= 4,464 used= 3,816 count= 9 avg_used= 424 block_cnt= 7 chunk_cnt= 7 mod=ContextsMap [MEMORY] hold= 4,464 used= 3,816 count= 9 avg_used= 424 block_cnt= 7 chunk_cnt= 7 mod=SequenceIdMap [MEMORY] hold= 4,464 used= 4,032 count= 6 avg_used= 672 block_cnt= 6 chunk_cnt= 6 mod=[T]ObWarningBuf [MEMORY] hold= 4,464 used= 3,816 count= 9 avg_used= 424 block_cnt= 7 chunk_cnt= 7 mod=PackStateMap [MEMORY] hold= 3,888 used= 3,816 count= 1 avg_used= 3,816 block_cnt= 1 chunk_cnt= 1 mod=RemMasterMap [MEMORY] hold= 3,888 used= 3,816 count= 1 avg_used= 3,816 block_cnt= 1 chunk_cnt= 1 mod=RecScheHisMap [MEMORY] hold= 3,840 used= 3,520 count= 5 avg_used= 704 block_cnt= 3 chunk_cnt= 3 mod=ScheMgrCacheMap [MEMORY] hold= 3,312 used= 2,880 count= 6 avg_used= 480 block_cnt= 6 chunk_cnt= 6 mod=TntResourceMgr [MEMORY] hold= 3,136 used= 3,072 count= 1 avg_used= 3,072 block_cnt= 1 chunk_cnt= 1 mod=TsMgr [MEMORY] hold= 2,768 used= 2,704 count= 1 avg_used= 2,704 block_cnt= 1 chunk_cnt= 1 mod=LoggerAlloc [MEMORY] hold= 1,760 used= 880 count= 10 avg_used= 88 block_cnt= 10 chunk_cnt= 5 mod=DestKAState [MEMORY] hold= 1,504 used= 1,432 count= 1 avg_used= 1,432 block_cnt= 1 chunk_cnt= 1 mod=GtsRequestRpc [MEMORY] hold= 1,408 used= 1,344 count= 1 avg_used= 1,344 block_cnt= 1 chunk_cnt= 1 mod=SchemaService [MEMORY] hold= 1,392 used= 1,328 count= 1 avg_used= 1,328 block_cnt= 1 chunk_cnt= 1 mod=GtsRpcProxy [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=IO_TENANT_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=INGRESS_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=IO_CHANNEL_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=TENANT_PLAN_MAP [MEMORY] hold= 1,152 used= 1,088 count= 1 avg_used= 1,088 block_cnt= 1 chunk_cnt= 1 mod=memdumpqueue [MEMORY] hold= 960 used= 280 count= 10 avg_used= 28 block_cnt= 3 chunk_cnt= 3 mod=KeepAliveServer [MEMORY] hold= 800 used= 336 count= 7 avg_used= 48 block_cnt= 7 chunk_cnt= 6 mod=[T]TSITabletTab [MEMORY] hold= 752 used= 336 count= 6 avg_used= 56 block_cnt= 6 chunk_cnt= 5 mod=UserResourceMgr [MEMORY] hold= 640 used= 280 count= 5 avg_used= 56 block_cnt= 5 chunk_cnt= 4 mod=PxTargetMgr [MEMORY] hold= 640 used= 280 count= 5 avg_used= 56 block_cnt= 5 chunk_cnt= 1 mod=Log [MEMORY] hold= 560 used= 14 count= 7 avg_used= 2 block_cnt= 6 chunk_cnt= 6 mod=[T]TSIUseWeak [MEMORY] hold= 480 used= 96 count= 6 avg_used= 16 block_cnt= 5 chunk_cnt= 5 mod=CreateEntity [MEMORY] hold= 416 used= 272 count= 2 avg_used= 136 block_cnt= 1 chunk_cnt= 1 mod=unknown [MEMORY] hold= 288 used= 216 count= 1 avg_used= 216 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 224 used= 160 count= 1 avg_used= 160 block_cnt= 1 chunk_cnt= 1 mod=OccamTimeGuard [MEMORY] hold= 192 used= 48 count= 2 avg_used= 24 block_cnt= 2 chunk_cnt= 2 mod=RemoteTask [MEMORY] hold= 80 used= 7 count= 1 avg_used= 7 block_cnt= 1 chunk_cnt= 1 mod=SqlExpr [MEMORY] hold= 318,519,120 used= 309,351,429 count= 56,912 avg_used= 5,435 mod=SUMMARY [2024-06-25 17:00:18.759123] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=34] [MEMORY] tenant_id= 500 ctx_id= GLIBC hold= 20,971,520 used= 13,250,080 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 12,935,968 used= 8,521,245 count= 60,966 avg_used= 139 block_cnt= 1,571 chunk_cnt= 10 mod=glibc_malloc [MEMORY] hold= 311,456 used= 89,560 count= 2,942 avg_used= 30 block_cnt= 42 chunk_cnt= 1 mod=Buffer [MEMORY] hold= 2,656 used= 1,222 count= 20 avg_used= 61 block_cnt= 5 chunk_cnt= 1 mod=XmlGlobal [MEMORY] hold= 13,250,080 used= 8,612,027 count= 63,928 avg_used= 134 mod=SUMMARY [2024-06-25 17:00:18.759130] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 500 ctx_id= CO_STACK hold= 165,675,008 used= 163,086,336 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 163,086,336 used= 162,782,976 count= 316 avg_used= 515,136 block_cnt= 316 chunk_cnt= 79 mod=CoStack [MEMORY] hold= 163,086,336 used= 162,782,976 count= 316 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.759138] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 500 ctx_id= LIBEASY hold= 8,388,608 used= 4,223,872 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 4,223,872 used= 3,403,736 count= 599 avg_used= 5,682 block_cnt= 136 chunk_cnt= 4 mod=OB_TEST2_PCODE [MEMORY] hold= 4,223,872 used= 3,403,736 count= 599 avg_used= 5,682 mod=SUMMARY [2024-06-25 17:00:18.759146] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 500 ctx_id= LOGGER_CTX_ID hold= 14,680,064 used= 14,565,376 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 10,403,840 used= 10,398,720 count= 5 avg_used= 2,079,744 block_cnt= 5 chunk_cnt= 5 mod=Logger [MEMORY] hold= 4,161,536 used= 4,159,488 count= 2 avg_used= 2,079,744 block_cnt= 2 chunk_cnt= 2 mod=ErrorLogger [MEMORY] hold= 14,565,376 used= 14,558,208 count= 7 avg_used= 2,079,744 mod=SUMMARY [2024-06-25 17:00:18.759165] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 500 ctx_id= RPC_CTX_ID hold= 10,485,760 used= 1,294,336 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 1,277,952 used= 1,267,968 count= 78 avg_used= 16,256 block_cnt= 78 chunk_cnt= 5 mod=RpcDefault [MEMORY] hold= 16,384 used= 16,256 count= 1 avg_used= 16,256 block_cnt= 1 chunk_cnt= 1 mod=OB_PX_TARGET_RE [MEMORY] hold= 1,294,336 used= 1,284,224 count= 79 avg_used= 16,256 mod=SUMMARY [2024-06-25 17:00:18.759177] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 500 ctx_id= PKT_NIO hold= 160,014,336 used= 122,969,280 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 70,679,744 used= 70,248,480 count= 384 avg_used= 182,938 block_cnt= 143 chunk_cnt= 40 mod=DEFAULT [MEMORY] hold= 23,658,496 used= 23,575,680 count= 180 avg_used= 130,976 block_cnt= 180 chunk_cnt= 29 mod=PKTC_INBUF [MEMORY] hold= 23,617,536 used= 23,575,680 count= 180 avg_used= 130,976 block_cnt= 180 chunk_cnt= 29 mod=PKTS_INBUF [MEMORY] hold= 1,474,560 used= 1,465,920 count= 90 avg_used= 16,288 block_cnt= 90 chunk_cnt= 30 mod=SERVER_CTX_CHUN [MEMORY] hold= 1,179,648 used= 1,172,736 count= 72 avg_used= 16,288 block_cnt= 72 chunk_cnt= 29 mod=CLIENT_CB_CHUNK [MEMORY] hold= 1,179,648 used= 1,172,736 count= 72 avg_used= 16,288 block_cnt= 72 chunk_cnt= 15 mod=CLIENT_REQ_CHUN [MEMORY] hold= 1,179,648 used= 1,172,736 count= 72 avg_used= 16,288 block_cnt= 72 chunk_cnt= 28 mod=SERVER_RESP_CHU [MEMORY] hold= 122,969,280 used= 122,383,968 count= 1,050 avg_used= 116,556 mod=SUMMARY [2024-06-25 17:00:18.759228] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 500 ctx_id= SCHEMA_SERVICE hold= 158,093,312 used= 139,296,224 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 68,745,344 used= 68,710,912 count= 43 avg_used= 1,597,928 block_cnt= 43 chunk_cnt= 40 mod=TenantSchemMgr [MEMORY] hold= 60,561,856 used= 60,052,014 count= 6,745 avg_used= 8,903 block_cnt= 1,027 chunk_cnt= 38 mod=SchemaSysCache [MEMORY] hold= 7,129,472 used= 7,109,672 count= 5 avg_used= 1,421,934 block_cnt= 5 chunk_cnt= 2 mod=SchemaIdVersion [MEMORY] hold= 884,736 used= 786,432 count= 12 avg_used= 65,536 block_cnt= 12 chunk_cnt= 4 mod=SchemaMgrCache [MEMORY] hold= 495,360 used= 380,392 count= 36 avg_used= 10,566 block_cnt= 36 chunk_cnt= 14 mod=ScheTablInfoVec [MEMORY] hold= 208,000 used= 206,336 count= 26 avg_used= 7,936 block_cnt= 26 chunk_cnt= 12 mod=ScheTablIdMap [MEMORY] hold= 159,360 used= 158,080 count= 20 avg_used= 7,904 block_cnt= 20 chunk_cnt= 10 mod=ScheTenaInfoVec [MEMORY] hold= 144,000 used= 142,848 count= 18 avg_used= 7,936 block_cnt= 18 chunk_cnt= 8 mod=ScheTablNameMap [MEMORY] hold= 137,024 used= 114,392 count= 13 avg_used= 8,799 block_cnt= 13 chunk_cnt= 6 mod=ScheIndeInfoVec [MEMORY] hold= 87,648 used= 86,944 count= 11 avg_used= 7,904 block_cnt= 11 chunk_cnt= 8 mod=ScheLobPInfoVec [MEMORY] hold= 87,648 used= 86,944 count= 11 avg_used= 7,904 block_cnt= 11 chunk_cnt= 5 mod=ScheLobMInfoVec [MEMORY] hold= 81,664 used= 80,320 count= 20 avg_used= 4,016 block_cnt= 19 chunk_cnt= 12 mod=SchemaKeystore [MEMORY] hold= 81,664 used= 80,320 count= 20 avg_used= 4,016 block_cnt= 20 chunk_cnt= 10 mod=SchemaSysVariab [MEMORY] hold= 79,680 used= 79,040 count= 10 avg_used= 7,904 block_cnt= 10 chunk_cnt= 7 mod=SchePrivDbPrivs [MEMORY] hold= 79,680 used= 79,040 count= 10 avg_used= 7,904 block_cnt= 10 chunk_cnt= 7 mod=ScheDbInfoVec [MEMORY] hold= 72,288 used= 71,712 count= 9 avg_used= 7,968 block_cnt= 9 chunk_cnt= 4 mod=ScheMgrInfoArra [MEMORY] hold= 48,896 used= 48,128 count= 12 avg_used= 4,010 block_cnt= 12 chunk_cnt= 5 mod=ScheIndeNameMap [MEMORY] hold= 17,888 used= 17,088 count= 12 avg_used= 1,424 block_cnt= 12 chunk_cnt= 8 mod=SchemaSequence [MEMORY] hold= 16,064 used= 15,808 count= 2 avg_used= 7,904 block_cnt= 2 chunk_cnt= 2 mod=SchePackInfoVec [MEMORY] hold= 16,064 used= 15,808 count= 2 avg_used= 7,904 block_cnt= 2 chunk_cnt= 2 mod=ScheRoutInfoVec [MEMORY] hold= 10,880 used= 10,240 count= 10 avg_used= 1,024 block_cnt= 10 chunk_cnt= 6 mod=ScheRoutNameMap [MEMORY] hold= 10,880 used= 10,240 count= 10 avg_used= 1,024 block_cnt= 10 chunk_cnt= 6 mod=SchePackIdMap [MEMORY] hold= 10,880 used= 10,240 count= 10 avg_used= 1,024 block_cnt= 9 chunk_cnt= 5 mod=SchePackNameMap [MEMORY] hold= 10,880 used= 10,240 count= 10 avg_used= 1,024 block_cnt= 10 chunk_cnt= 6 mod=ScheRoutIdMap [MEMORY] hold= 10,880 used= 10,240 count= 10 avg_used= 1,024 block_cnt= 10 chunk_cnt= 5 mod=HiddenTblNames [MEMORY] hold= 8,288 used= 5,120 count= 40 avg_used= 128 block_cnt= 25 chunk_cnt= 11 mod=ScheLabeSeCompo [MEMORY] hold= 6,240 used= 3,840 count= 30 avg_used= 128 block_cnt= 20 chunk_cnt= 9 mod=ScheLabeSePolic [MEMORY] hold= 6,176 used= 3,840 count= 30 avg_used= 128 block_cnt= 23 chunk_cnt= 10 mod=ScheLabeSeLabel [MEMORY] hold= 5,760 used= 5,120 count= 10 avg_used= 512 block_cnt= 10 chunk_cnt= 6 mod=ScheUdtNameMap [MEMORY] hold= 5,760 used= 5,120 count= 10 avg_used= 512 block_cnt= 10 chunk_cnt= 6 mod=ScheUdtIdMap [MEMORY] hold= 4,096 used= 2,560 count= 20 avg_used= 128 block_cnt= 17 chunk_cnt= 8 mod=ScheLabSeUserLe [MEMORY] hold= 4,032 used= 2,560 count= 20 avg_used= 128 block_cnt= 13 chunk_cnt= 7 mod=RLS_CONTEXT_MGR [MEMORY] hold= 4,032 used= 2,560 count= 20 avg_used= 128 block_cnt= 13 chunk_cnt= 6 mod=RLS_GROUP_MGR [MEMORY] hold= 4,032 used= 2,560 count= 20 avg_used= 128 block_cnt= 13 chunk_cnt= 6 mod=RLS_POLICY_MGR [MEMORY] hold= 4,000 used= 2,560 count= 20 avg_used= 128 block_cnt= 15 chunk_cnt= 8 mod=DIRECTORY_MGR [MEMORY] hold= 3,904 used= 2,560 count= 20 avg_used= 128 block_cnt= 11 chunk_cnt= 7 mod=SchemaProfile [MEMORY] hold= 3,904 used= 2,560 count= 20 avg_used= 128 block_cnt= 12 chunk_cnt= 6 mod=SchemaSynonym [MEMORY] hold= 3,904 used= 2,560 count= 20 avg_used= 128 block_cnt= 13 chunk_cnt= 8 mod=DBLINK_MGR [MEMORY] hold= 3,840 used= 2,560 count= 20 avg_used= 128 block_cnt= 12 chunk_cnt= 9 mod=ScheOutlSqlMap [MEMORY] hold= 3,712 used= 3,072 count= 10 avg_used= 307 block_cnt= 9 chunk_cnt= 6 mod=ScheDataNameMap [MEMORY] hold= 3,280 used= 2,920 count= 5 avg_used= 584 block_cnt= 5 chunk_cnt= 4 mod=TenaScheMemMgr [MEMORY] hold= 3,280 used= 2,920 count= 5 avg_used= 584 block_cnt= 4 chunk_cnt= 3 mod=TenSchMemMgrFoL [MEMORY] hold= 2,080 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 7 mod=SchemaTablespac [MEMORY] hold= 1,984 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=SchemaUdf [MEMORY] hold= 1,984 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=SchemaSecurAudi [MEMORY] hold= 1,952 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=PRIV_ROUTINE [MEMORY] hold= 1,952 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 7 mod=ScheConsNameMap [MEMORY] hold= 1,952 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 7 mod=ScheForKeyNamMa [MEMORY] hold= 1,952 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 8 mod=ScheAuxVpNameVe [MEMORY] hold= 1,952 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=ScheTrigNameMap [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=SchePriTabPriMa [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 9 chunk_cnt= 6 mod=SchePriObjPriMa [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 9 chunk_cnt= 4 mod=ScheTrigIdMap [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 9 chunk_cnt= 7 mod=ScheOutlNameMap [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 7 mod=ScheOutlIdMap [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 6 mod=MockFkParentTab [MEMORY] hold= 1,920 used= 1,280 count= 10 avg_used= 128 block_cnt= 10 chunk_cnt= 5 mod=SchemaContext [MEMORY] hold= 139,296,224 used= 138,447,622 count= 7,557 avg_used= 18,320 mod=SUMMARY [2024-06-25 17:00:18.759254] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=16] [MEMORY] tenant_id= 500 ctx_id= UNEXPECTED_IN_500 hold= 2,473,402,368 used= 2,440,823,872 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 1,745,096,880 used= 1,744,830,568 count= 14 avg_used= 124,630,754 block_cnt= 14 chunk_cnt= 14 mod=CACHE_MAP_BKT [MEMORY] hold= 268,546,944 used= 268,444,576 count= 7 avg_used= 38,349,225 block_cnt= 7 chunk_cnt= 6 mod=FixeSizeBlocAll [MEMORY] hold= 201,347,072 used= 201,326,616 count= 1 avg_used= 201,326,616 block_cnt= 1 chunk_cnt= 1 mod=CACHE_MAP_LOCK [MEMORY] hold= 78,606,336 used= 78,587,160 count= 1 avg_used= 78,587,160 block_cnt= 1 chunk_cnt= 1 mod=CACHE_MB_HANDLE [MEMORY] hold= 64,519,168 used= 64,471,552 count= 292 avg_used= 220,792 block_cnt= 292 chunk_cnt= 57 mod=BlockMap [MEMORY] hold= 34,734,080 used= 34,611,840 count= 40 avg_used= 865,296 block_cnt= 40 chunk_cnt= 23 mod=di_tenant_cache [MEMORY] hold= 17,329,216 used= 8,968,960 count= 1,093 avg_used= 8,205 block_cnt= 1,093 chunk_cnt= 9 mod=StorageLoggerM [MEMORY] hold= 15,550,048 used= 13,817,664 count= 398 avg_used= 34,717 block_cnt= 230 chunk_cnt= 11 mod=OccamThreadPool [MEMORY] hold= 12,042,240 used= 11,947,880 count= 5 avg_used= 2,389,576 block_cnt= 5 chunk_cnt= 5 mod=TenantConfig [MEMORY] hold= 3,036,992 used= 2,917,447 count= 157 avg_used= 18,582 block_cnt= 126 chunk_cnt= 23 mod=CommonNetwork [MEMORY] hold= 14,896 used= 2,656 count= 166 avg_used= 16 block_cnt= 16 chunk_cnt= 3 mod=ConfigChecker [MEMORY] hold= 2,440,823,872 used= 2,429,926,919 count= 2,174 avg_used= 1,117,721 mod=SUMMARY [2024-06-25 17:00:18.759268] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=6] [MEMORY] tenant: 508, limit: 1,073,741,824 hold: 40,296,448 rpc_hold: 0 cache_hold: 0 cache_used: 0 cache_item_count: 0 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 13,033,472 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 27,262,976 limit= 9,223,372,036,854,775,807 [2024-06-25 17:00:18.759284] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 508 ctx_id= DEFAULT_CTX_ID hold= 13,033,472 used= 11,676,016 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 3,833,856 used= 3,423,264 count= 52 avg_used= 65,832 block_cnt= 52 chunk_cnt= 3 mod=[T]ObSessionDIB [MEMORY] hold= 3,760,128 used= 3,342,336 count= 51 avg_used= 65,536 block_cnt= 51 chunk_cnt= 4 mod=[T]char [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 413,712 used= 410,448 count= 51 avg_used= 8,048 block_cnt= 51 chunk_cnt= 4 mod=[T]ObPerfEventR [MEMORY] hold= 235,008 used= 231,744 count= 51 avg_used= 4,544 block_cnt= 51 chunk_cnt= 4 mod=[T]ObTraceEvent [MEMORY] hold= 98,304 used= 94,848 count= 1 avg_used= 94,848 block_cnt= 1 chunk_cnt= 1 mod=ResourceGroup [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DynamicFactor [MEMORY] hold= 37,536 used= 34,272 count= 51 avg_used= 672 block_cnt= 51 chunk_cnt= 4 mod=[T]ObWarningBuf [MEMORY] hold= 24,960 used= 21,632 count= 52 avg_used= 416 block_cnt= 7 chunk_cnt= 2 mod=OMT_Worker [MEMORY] hold= 14,144 used= 6,656 count= 104 avg_used= 64 block_cnt= 7 chunk_cnt= 2 mod=Coro [MEMORY] hold= 4,992 used= 1,248 count= 52 avg_used= 24 block_cnt= 7 chunk_cnt= 2 mod=[T]MemoryContex [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 11,676,016 used= 10,793,616 count= 469 avg_used= 23,014 mod=SUMMARY [2024-06-25 17:00:18.759305] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=6] [MEMORY] tenant_id= 508 ctx_id= CO_STACK hold= 27,262,976 used= 26,836,992 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 26,836,992 used= 26,787,072 count= 52 avg_used= 515,136 block_cnt= 52 chunk_cnt= 13 mod=CoStack [MEMORY] hold= 26,836,992 used= 26,787,072 count= 52 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.759391] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant: 1, limit: 40,399,536,128 hold: 17,737,887,744 rpc_hold: 0 cache_hold: 14,854,127,616 cache_used: 14,854,127,616 cache_item_count: 7,083 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 2,015,518,720 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MEMSTORE_CTX_ID hold_bytes= 285,212,672 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= TRANS_CTX_MGR_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PLAN_CACHE_CTX_ID hold_bytes= 312,475,648 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= GLIBC hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 146,800,640 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= META_OBJ_CTX_ID hold_bytes= 20,971,520 limit= 8,079,907,220 [MEMORY] ctx_id= TX_CALLBACK_CTX_ID hold_bytes= 20,971,520 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= LOB_CTX_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PS_CACHE_CTX_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= TX_DATA_TABLE hold_bytes= 69,226,496 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MDS_DATA_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY][PM] tid= 62760 used= 0 hold= 4,194,304 pm=0x2b3a5aad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62761 used= 0 hold= 2,097,152 pm=0x2b3a5ab52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62764 used= 0 hold= 4,194,304 pm=0x2b3a5acd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62769 used= 0 hold= 4,194,304 pm=0x2b3a5b152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62770 used= 0 hold= 4,194,304 pm=0x2b3a5b1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62771 used= 0 hold= 4,194,304 pm=0x2b3a5b256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62772 used= 0 hold= 4,194,304 pm=0x2b3a5b2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62773 used= 0 hold= 4,194,304 pm=0x2b3a5b352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62774 used= 0 hold= 4,194,304 pm=0x2b3a5b3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62775 used= 0 hold= 4,194,304 pm=0x2b3a5b656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62776 used= 0 hold= 4,194,304 pm=0x2b3a5b6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62777 used= 0 hold= 4,194,304 pm=0x2b3a5b752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62778 used= 0 hold= 4,194,304 pm=0x2b3a5b7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62779 used= 0 hold= 4,194,304 pm=0x2b3a5b856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62780 used= 0 hold= 4,194,304 pm=0x2b3a5b8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62781 used= 0 hold= 4,194,304 pm=0x2b3a5b952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62782 used= 0 hold= 4,194,304 pm=0x2b3a5b9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62783 used= 0 hold= 4,194,304 pm=0x2b3a5ba56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62784 used= 0 hold= 4,194,304 pm=0x2b3a5bad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62791 used= 0 hold= 2,097,152 pm=0x2b3a5bb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62792 used= 0 hold= 2,097,152 pm=0x2b3a5bbd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62793 used= 0 hold= 2,097,152 pm=0x2b3a5c656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62794 used= 0 hold= 2,097,152 pm=0x2b3a5c6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62913 used= 0 hold= 4,194,304 pm=0x2b3a66fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62914 used= 0 hold= 4,194,304 pm=0x2b3a7e856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62915 used= 0 hold= 4,194,304 pm=0x2b3a7e8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62916 used= 0 hold= 4,194,304 pm=0x2b3a7e952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62917 used= 0 hold= 4,194,304 pm=0x2b3a7e9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62918 used= 0 hold= 4,194,304 pm=0x2b3a7ea56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62919 used= 0 hold= 4,194,304 pm=0x2b3a7ead4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62920 used= 0 hold= 4,194,304 pm=0x2b3a7eb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62921 used= 0 hold= 4,194,304 pm=0x2b3a7ebd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62922 used= 0 hold= 4,194,304 pm=0x2b3a7ec56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62923 used= 0 hold= 4,194,304 pm=0x2b3a7ecd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62924 used= 0 hold= 4,194,304 pm=0x2b3a7ed52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62925 used= 0 hold= 4,194,304 pm=0x2b3a7edd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62926 used= 0 hold= 4,194,304 pm=0x2b3a7ee56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62927 used= 0 hold= 4,194,304 pm=0x2b3a7eed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62928 used= 0 hold= 4,194,304 pm=0x2b3a7ef52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62929 used= 2,145,344 hold= 4,194,304 pm=0x2b3a7efd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62930 used= 0 hold= 4,194,304 pm=0x2b3a7f256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62931 used= 0 hold= 4,194,304 pm=0x2b3a7f2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62932 used= 0 hold= 4,194,304 pm=0x2b3a7f352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62933 used= 0 hold= 4,194,304 pm=0x2b3a7f3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62934 used= 0 hold= 4,194,304 pm=0x2b3a7f456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62935 used= 0 hold= 4,194,304 pm=0x2b3a7f4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 62936 used= 0 hold= 4,194,304 pm=0x2b3a7f552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77744 used= 0 hold= 2,097,152 pm=0x2bd04c1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77745 used= 0 hold= 2,097,152 pm=0x2bd04c256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77746 used= 0 hold= 4,194,304 pm=0x2bd04c2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77747 used= 0 hold= 4,194,304 pm=0x2bd04c352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77748 used= 0 hold= 4,194,304 pm=0x2bd04c3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77749 used= 0 hold= 4,194,304 pm=0x2bd04c456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77750 used= 0 hold= 4,194,304 pm=0x2bd04c4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77751 used= 0 hold= 4,194,304 pm=0x2bd04c552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77752 used= 0 hold= 4,194,304 pm=0x2bd04c5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77753 used= 0 hold= 4,194,304 pm=0x2bd04c656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77754 used= 0 hold= 4,194,304 pm=0x2bd04c6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77755 used= 0 hold= 4,194,304 pm=0x2bd04c752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77756 used= 0 hold= 4,194,304 pm=0x2bd04c7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77757 used= 0 hold= 4,194,304 pm=0x2bd04c856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77758 used= 0 hold= 4,194,304 pm=0x2bd04c8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77759 used= 0 hold= 4,194,304 pm=0x2bd04c952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77760 used= 0 hold= 4,194,304 pm=0x2bd04c9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77761 used= 0 hold= 4,194,304 pm=0x2bd050456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77762 used= 0 hold= 4,194,304 pm=0x2bd0504d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77763 used= 0 hold= 4,194,304 pm=0x2bd050552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77764 used= 0 hold= 4,194,304 pm=0x2bd0505d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 77765 used= 0 hold= 4,194,304 pm=0x2bd050656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= summary used= 2,145,344 hold= 274,726,912 [2024-06-25 17:00:18.759543] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=11] [MEMORY] tenant_id= 1 ctx_id= DEFAULT_CTX_ID hold= 2,015,518,720 used= 1,617,364,480 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 847,839,232 used= 847,436,480 count= 371 avg_used= 2,284,195 block_cnt= 371 chunk_cnt= 371 mod=MysqlRequesReco [MEMORY] hold= 132,341,760 used= 117,637,120 count= 1,795 avg_used= 65,536 block_cnt= 1,795 chunk_cnt= 177 mod=MinorMergeMgr [MEMORY] hold= 76,988,416 used= 76,950,528 count= 37 avg_used= 2,079,744 block_cnt= 37 chunk_cnt= 37 mod=CACHE_MAP_NODE [MEMORY] hold= 76,742,656 used= 76,002,560 count= 37 avg_used= 2,054,123 block_cnt= 37 chunk_cnt= 37 mod=SqlPlanMon [MEMORY] hold= 62,373,888 used= 55,443,456 count= 846 avg_used= 65,536 block_cnt= 846 chunk_cnt= 165 mod=MajorMergeMgr [MEMORY] hold= 41,963,520 used= 41,943,040 count= 1 avg_used= 41,943,040 block_cnt= 1 chunk_cnt= 1 mod=LogGroupBuffer [MEMORY] hold= 39,350,272 used= 39,321,760 count= 2 avg_used= 19,660,880 block_cnt= 2 chunk_cnt= 2 mod=HashBuckDmId [MEMORY] hold= 37,769,216 used= 37,748,856 count= 1 avg_used= 37,748,856 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmChe [MEMORY] hold= 31,477,760 used= 31,458,304 count= 1 avg_used= 31,458,304 block_cnt= 1 chunk_cnt= 1 mod=ash_list [MEMORY] hold= 21,233,664 used= 18,959,616 count= 288 avg_used= 65,832 block_cnt= 288 chunk_cnt= 54 mod=[T]ObSessionDIB [MEMORY] hold= 18,874,368 used= 16,887,552 count= 256 avg_used= 65,967 block_cnt= 256 chunk_cnt= 69 mod=SqlDtlBuf [MEMORY] hold= 13,647,872 used= 13,603,118 count= 7 avg_used= 1,943,302 block_cnt= 7 chunk_cnt= 3 mod=LOCALDEVICE [MEMORY] hold= 13,074,080 used= 13,049,736 count= 60 avg_used= 217,495 block_cnt= 60 chunk_cnt= 17 mod=TabletMap [MEMORY] hold= 12,808,192 used= 12,763,552 count= 42 avg_used= 303,894 block_cnt= 42 chunk_cnt= 35 mod=IoControl [MEMORY] hold= 10,428,416 used= 10,387,456 count= 2 avg_used= 5,193,728 block_cnt= 2 chunk_cnt= 2 mod=ServerObjecPool [MEMORY] hold= 9,484,544 used= 9,332,736 count= 2,352 avg_used= 3,968 block_cnt= 1,182 chunk_cnt= 102 mod=CkpDgn [MEMORY] hold= 8,895,616 used= 8,810,249 count= 79 avg_used= 111,522 block_cnt= 27 chunk_cnt= 7 mod=OmtTenant [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=ArcFetchQueue [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=RCSrv [MEMORY] hold= 8,257,536 used= 8,240,064 count= 2 avg_used= 4,120,032 block_cnt= 2 chunk_cnt= 2 mod=FreeTbltStream [MEMORY] hold= 8,151,040 used= 8,131,072 count= 1 avg_used= 8,131,072 block_cnt= 1 chunk_cnt= 1 mod=TransService [MEMORY] hold= 7,881,232 used= 7,816,400 count= 1,007 avg_used= 7,762 block_cnt= 1,007 chunk_cnt= 134 mod=TLDecoderAlloc [MEMORY] hold= 7,827,648 used= 7,704,288 count= 1,706 avg_used= 4,515 block_cnt= 1,645 chunk_cnt= 161 mod=WorkerMap [MEMORY] hold= 7,815,168 used= 6,815,744 count= 122 avg_used= 55,866 block_cnt= 122 chunk_cnt= 44 mod=[T]char [MEMORY] hold= 5,554,624 used= 5,508,944 count= 707 avg_used= 7,792 block_cnt= 707 chunk_cnt= 97 mod=CkptDgnMemNode [MEMORY] hold= 4,739,072 used= 4,718,712 count= 1 avg_used= 4,718,712 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMonMap [MEMORY] hold= 4,161,536 used= 4,159,488 count= 2 avg_used= 2,079,744 block_cnt= 2 chunk_cnt= 2 mod=SqlExecutor [MEMORY] hold= 4,072,528 used= 2,305,320 count= 24,367 avg_used= 94 block_cnt= 63 chunk_cnt= 49 mod=TLDecoderCtx [MEMORY] hold= 4,066,688 used= 4,023,552 count= 476 avg_used= 8,452 block_cnt= 476 chunk_cnt= 134 mod=ParamCharsType [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMemCU [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 12 mod=CkptDgnMem [MEMORY] hold= 3,723,264 used= 3,698,688 count= 2 avg_used= 1,849,344 block_cnt= 2 chunk_cnt= 2 mod=LogDIOAligned [MEMORY] hold= 3,145,728 used= 2,228,224 count= 128 avg_used= 17,408 block_cnt= 128 chunk_cnt= 6 mod=SqlDtlQueue [MEMORY] hold= 2,981,888 used= 2,960,672 count= 13 avg_used= 227,744 block_cnt= 13 chunk_cnt= 12 mod=SQLSessionInfo [MEMORY] hold= 2,756,608 used= 2,720,005 count= 3 avg_used= 906,668 block_cnt= 3 chunk_cnt= 3 mod=SqlDtl1stBuf [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=TransTimeWheel [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=XATimeWheel [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=MediumTabletMap [MEMORY] hold= 2,375,680 used= 2,359,536 count= 2 avg_used= 1,179,768 block_cnt= 2 chunk_cnt= 2 mod=HashBuckLCSta [MEMORY] hold= 2,248,704 used= 2,228,224 count= 1 avg_used= 2,228,224 block_cnt= 1 chunk_cnt= 1 mod=LogIOCb [MEMORY] hold= 2,169,056 used= 408,600 count= 24,952 avg_used= 16 block_cnt= 266 chunk_cnt= 2 mod=Number [MEMORY] hold= 2,097,152 used= 2,093,056 count= 32 avg_used= 65,408 block_cnt= 32 chunk_cnt= 32 mod=StorMetaCacheIO [MEMORY] hold= 2,080,768 used= 2,079,744 count= 1 avg_used= 2,079,744 block_cnt= 1 chunk_cnt= 1 mod=LogReplayTask [MEMORY] hold= 1,966,080 used= 1,962,240 count= 30 avg_used= 65,408 block_cnt= 30 chunk_cnt= 7 mod=MemTblObj [MEMORY] hold= 1,777,664 used= 1,770,696 count= 1 avg_used= 1,770,696 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 1,670,592 used= 1,663,936 count= 7 avg_used= 237,705 block_cnt= 7 chunk_cnt= 2 mod=PoolFreeList [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=T3MBucket [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 1,433,600 used= 1,284,640 count= 35 avg_used= 36,704 block_cnt= 35 chunk_cnt= 31 mod=CommSysVarFac [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPlanCac [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPsInfo [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPsCache [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=RewriteRuleMap [MEMORY] hold= 1,048,576 used= 546,816 count= 64 avg_used= 8,544 block_cnt= 64 chunk_cnt= 26 mod=[T]ObDSActionAr [MEMORY] hold= 1,024,000 used= 1,015,808 count= 1 avg_used= 1,015,808 block_cnt= 1 chunk_cnt= 1 mod=CommitVersions [MEMORY] hold= 958,464 used= 950,272 count= 1 avg_used= 950,272 block_cnt= 1 chunk_cnt= 1 mod=IOWorkerLQ [MEMORY] hold= 933,888 used= 931,072 count= 1 avg_used= 931,072 block_cnt= 1 chunk_cnt= 1 mod=ArcSenderQueue [MEMORY] hold= 829,632 used= 804,864 count= 258 avg_used= 3,119 block_cnt= 130 chunk_cnt= 5 mod=LSMap [MEMORY] hold= 802,944 used= 786,432 count= 256 avg_used= 3,072 block_cnt= 128 chunk_cnt= 3 mod=TabletToLS [MEMORY] hold= 802,816 used= 800,000 count= 1 avg_used= 800,000 block_cnt= 1 chunk_cnt= 1 mod=SqlFltSpanRec [MEMORY] hold= 786,560 used= 524,352 count= 33 avg_used= 15,889 block_cnt= 33 chunk_cnt= 4 mod=LogAlloc [MEMORY] hold= 730,608 used= 724,656 count= 93 avg_used= 7,792 block_cnt= 93 chunk_cnt= 58 mod=CkptDgnCUNode [MEMORY] hold= 720,896 used= 712,704 count= 1 avg_used= 712,704 block_cnt= 1 chunk_cnt= 1 mod=DiskIteratorSto [MEMORY] hold= 673,296 used= 667,984 count= 83 avg_used= 8,048 block_cnt= 83 chunk_cnt= 46 mod=[T]ObPerfEventR [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 663,552 used= 655,360 count= 1 avg_used= 655,360 block_cnt= 1 chunk_cnt= 1 mod=FetchLog [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTR [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DupTbLease [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MultiVersionGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTF [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=OBJLockGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MdsT [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=ElectTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DetectorTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=FrzTrigger [MEMORY] hold= 608,000 used= 603,136 count= 76 avg_used= 7,936 block_cnt= 76 chunk_cnt= 45 mod=NotParamInfo [MEMORY] hold= 598,016 used= 590,232 count= 1 avg_used= 590,232 block_cnt= 1 chunk_cnt= 1 mod=DagNetIdMap [MEMORY] hold= 576,000 used= 571,392 count= 72 avg_used= 7,936 block_cnt= 72 chunk_cnt= 43 mod=SqlSession [MEMORY] hold= 524,288 used= 523,264 count= 8 avg_used= 65,408 block_cnt= 8 chunk_cnt= 5 mod=MemTblMgrObj [MEMORY] hold= 491,520 used= 474,240 count= 5 avg_used= 94,848 block_cnt= 5 chunk_cnt= 5 mod=ResourceGroup [MEMORY] hold= 451,584 used= 445,312 count= 98 avg_used= 4,544 block_cnt= 98 chunk_cnt= 46 mod=[T]ObTraceEvent [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ReplaySrv [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ApplySrv [MEMORY] hold= 385,024 used= 377,120 count= 1 avg_used= 377,120 block_cnt= 1 chunk_cnt= 1 mod=ClogGe [MEMORY] hold= 385,024 used= 375,520 count= 2 avg_used= 187,760 block_cnt= 2 chunk_cnt= 2 mod=bf_queue [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=DmlStatHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=ColUsagHashMap [MEMORY] hold= 280,128 used= 277,760 count= 35 avg_used= 7,936 block_cnt= 35 chunk_cnt= 24 mod=SqlSessiVarMap [MEMORY] hold= 280,000 used= 277,760 count= 35 avg_used= 7,936 block_cnt= 35 chunk_cnt= 32 mod=PlanVaIdx [MEMORY] hold= 256,128 used= 253,952 count= 32 avg_used= 7,936 block_cnt= 32 chunk_cnt= 30 mod=LSIter [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagNetMap [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagMap [MEMORY] hold= 200,704 used= 196,608 count= 64 avg_used= 3,072 block_cnt= 32 chunk_cnt= 3 mod=TxCtxMgr [MEMORY] hold= 169,344 used= 166,656 count= 42 avg_used= 3,968 block_cnt= 22 chunk_cnt= 14 mod=TinyAllocator [MEMORY] hold= 147,456 used= 139,264 count= 1 avg_used= 139,264 block_cnt= 1 chunk_cnt= 1 mod=RFLTaskQueue [MEMORY] hold= 141,696 used= 140,544 count= 18 avg_used= 7,808 block_cnt= 18 chunk_cnt= 15 mod=HashNodeLCSta [MEMORY] hold= 122,880 used= 110,608 count= 2 avg_used= 55,304 block_cnt= 2 chunk_cnt= 1 mod=TabletIDSetBkt [MEMORY] hold= 114,688 used= 106,496 count= 1 avg_used= 106,496 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlMgr [MEMORY] hold= 98,304 used= 94,976 count= 1 avg_used= 94,976 block_cnt= 1 chunk_cnt= 1 mod=PxPool [MEMORY] hold= 90,112 used= 84,448 count= 1 avg_used= 84,448 block_cnt= 1 chunk_cnt= 1 mod=LSSvr [MEMORY] hold= 90,112 used= 82,048 count= 1 avg_used= 82,048 block_cnt= 1 chunk_cnt= 1 mod=MetaMemMgr [MEMORY] hold= 74,160 used= 35,568 count= 525 avg_used= 67 block_cnt= 185 chunk_cnt= 38 mod=Coro [MEMORY] hold= 73,728 used= 72,736 count= 1 avg_used= 72,736 block_cnt= 1 chunk_cnt= 1 mod=LogSharedQueueT [MEMORY] hold= 73,728 used= 66,176 count= 1 avg_used= 66,176 block_cnt= 1 chunk_cnt= 1 mod=Rpc [MEMORY] hold= 73,728 used= 65,536 count= 1 avg_used= 65,536 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoMgr [MEMORY] hold= 72,240 used= 1,024 count= 896 avg_used= 1 block_cnt= 10 chunk_cnt= 1 mod=CharsetUtil [MEMORY] hold= 70,848 used= 70,272 count= 9 avg_used= 7,808 block_cnt= 9 chunk_cnt= 9 mod=HashNodePlanCac [MEMORY] hold= 70,848 used= 70,272 count= 9 avg_used= 7,808 block_cnt= 9 chunk_cnt= 8 mod=TabletIDSetNode [MEMORY] hold= 67,920 used= 61,824 count= 92 avg_used= 672 block_cnt= 67 chunk_cnt= 38 mod=[T]ObWarningBuf [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=FlushLog [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=DEVICE_MANAGER [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxDataMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxCtxMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LockMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=FlushMeta [MEMORY] hold= 59,040 used= 50,184 count= 123 avg_used= 408 block_cnt= 120 chunk_cnt= 57 mod=[T]ObTLDecoderC [MEMORY] hold= 49,152 used= 48,768 count= 1 avg_used= 48,768 block_cnt= 1 chunk_cnt= 1 mod=palf_env [MEMORY] hold= 48,832 used= 44,840 count= 2 avg_used= 22,420 block_cnt= 2 chunk_cnt= 2 mod=DynamicFactor [MEMORY] hold= 42,864 used= 37,024 count= 89 avg_used= 416 block_cnt= 37 chunk_cnt= 13 mod=OMT_Worker [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=HB_SERVICE [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=Autoincrement [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisBkt [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 1 mod=ObDMMDL [MEMORY] hold= 35,200 used= 34,560 count= 10 avg_used= 3,456 block_cnt= 10 chunk_cnt= 6 mod=CACHE_INST [MEMORY] hold= 32,768 used= 24,688 count= 2 avg_used= 12,344 block_cnt= 2 chunk_cnt= 1 mod=TLD_ClientTask [MEMORY] hold= 32,768 used= 17,408 count= 2 avg_used= 8,704 block_cnt= 2 chunk_cnt= 2 mod=ObLogEXTTP [MEMORY] hold= 32,768 used= 25,600 count= 1 avg_used= 25,600 block_cnt= 1 chunk_cnt= 1 mod=TSQLSessionMgr [MEMORY] hold= 32,000 used= 31,744 count= 4 avg_used= 7,936 block_cnt= 4 chunk_cnt= 4 mod=MdsTableMgr [MEMORY] hold= 28,400 used= 6,912 count= 288 avg_used= 24 block_cnt= 116 chunk_cnt= 47 mod=[T]MemoryContex [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 24,240 used= 20,136 count= 2 avg_used= 10,068 block_cnt= 2 chunk_cnt= 2 mod=TabletStats [MEMORY] hold= 23,280 used= 20,980 count= 35 avg_used= 599 block_cnt= 33 chunk_cnt= 24 mod=SqlSessiQuerSql [MEMORY] hold= 22,848 used= 21,504 count= 21 avg_used= 1,024 block_cnt= 20 chunk_cnt= 5 mod=TGTimer [MEMORY] hold= 19,648 used= 16,832 count= 44 avg_used= 382 block_cnt= 41 chunk_cnt= 37 mod=NegParamIdex [MEMORY] hold= 17,504 used= 14,840 count= 35 avg_used= 424 block_cnt= 34 chunk_cnt= 24 mod=SequenceMap [MEMORY] hold= 17,392 used= 14,840 count= 35 avg_used= 424 block_cnt= 33 chunk_cnt= 25 mod=PackStateMap [MEMORY] hold= 17,376 used= 14,840 count= 35 avg_used= 424 block_cnt= 33 chunk_cnt= 28 mod=ContextsMap [MEMORY] hold= 17,376 used= 14,840 count= 35 avg_used= 424 block_cnt= 35 chunk_cnt= 26 mod=SequenceIdMap [MEMORY] hold= 16,768 used= 8,448 count= 3 avg_used= 2,816 block_cnt= 2 chunk_cnt= 2 mod=BaseLogWriter [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TableCtxMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=backupTaskSched [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TblCtxIMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_AssignedMgr [MEMORY] hold= 16,384 used= 8,448 count= 1 avg_used= 8,448 block_cnt= 1 chunk_cnt= 1 mod=PalfEnv [MEMORY] hold= 16,384 used= 11,584 count= 1 avg_used= 11,584 block_cnt= 1 chunk_cnt= 1 mod=LogApplyStatus [MEMORY] hold= 16,384 used= 13,696 count= 1 avg_used= 13,696 block_cnt= 1 chunk_cnt= 1 mod=LogReplayStatus [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 10,248 count= 1 avg_used= 10,248 block_cnt= 1 chunk_cnt= 1 mod=TabletSvrBucket [MEMORY] hold= 15,936 used= 15,808 count= 2 avg_used= 7,904 block_cnt= 2 chunk_cnt= 2 mod=SqlSessionSbloc [MEMORY] hold= 12,576 used= 12,440 count= 2 avg_used= 6,220 block_cnt= 2 chunk_cnt= 2 mod=PxPoolBkt [MEMORY] hold= 10,240 used= 10,112 count= 2 avg_used= 5,056 block_cnt= 1 chunk_cnt= 1 mod=Scheduler [MEMORY] hold= 9,536 used= 6,016 count= 55 avg_used= 109 block_cnt= 55 chunk_cnt= 34 mod=NotParamIdex [MEMORY] hold= 8,192 used= 7,936 count= 4 avg_used= 1,984 block_cnt= 2 chunk_cnt= 2 mod=CreateContext [MEMORY] hold= 8,192 used= 8,056 count= 2 avg_used= 4,028 block_cnt= 2 chunk_cnt= 2 mod=StoLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=APPLY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LockWaitMgr [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LSLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ShareBlksMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MdsDebugMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=REPLAY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MatchLsnMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ls_id [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LCLSender [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LsElectRefInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DASIDCache [MEMORY] hold= 7,968 used= 7,904 count= 1 avg_used= 7,904 block_cnt= 1 chunk_cnt= 1 mod=PsSessionInfo [MEMORY] hold= 7,872 used= 7,808 count= 1 avg_used= 7,808 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoNode [MEMORY] hold= 7,872 used= 7,808 count= 1 avg_used= 7,808 block_cnt= 1 chunk_cnt= 1 mod=HashNodePsInfo [MEMORY] hold= 7,856 used= 7,792 count= 1 avg_used= 7,792 block_cnt= 1 chunk_cnt= 1 mod=ColUsagNode [MEMORY] hold= 7,792 used= 7,728 count= 1 avg_used= 7,728 block_cnt= 1 chunk_cnt= 1 mod=HashNodePsCache [MEMORY] hold= 6,416 used= 6,216 count= 3 avg_used= 2,072 block_cnt= 2 chunk_cnt= 2 mod=DetectManager [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResGrpIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResRuleIdMap [MEMORY] hold= 4,160 used= 4,096 count= 1 avg_used= 4,096 block_cnt= 1 chunk_cnt= 1 mod=MacroFile [MEMORY] hold= 3,776 used= 3,712 count= 1 avg_used= 3,712 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlDfc [MEMORY] hold= 3,136 used= 3,072 count= 1 avg_used= 3,072 block_cnt= 1 chunk_cnt= 1 mod=LockMemtableMgr [MEMORY] hold= 2,576 used= 64 count= 32 avg_used= 2 block_cnt= 31 chunk_cnt= 22 mod=[T]TSIUseWeak [MEMORY] hold= 2,336 used= 1,792 count= 8 avg_used= 224 block_cnt= 3 chunk_cnt= 3 mod=LogIOTask [MEMORY] hold= 2,240 used= 1,920 count= 5 avg_used= 384 block_cnt= 5 chunk_cnt= 5 mod=TableLockNode [MEMORY] hold= 2,224 used= 1,664 count= 8 avg_used= 208 block_cnt= 8 chunk_cnt= 7 mod=[T]ObMacroBlock [MEMORY] hold= 1,920 used= 1,800 count= 1 avg_used= 1,800 block_cnt= 1 chunk_cnt= 1 mod=PxResMgr [MEMORY] hold= 1,920 used= 1,856 count= 1 avg_used= 1,856 block_cnt= 1 chunk_cnt= 1 mod=LogIOWS [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=MSTXCTX [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TabStatMgrLock [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TXBFRetryId [MEMORY] hold= 1,408 used= 1,280 count= 2 avg_used= 640 block_cnt= 1 chunk_cnt= 1 mod=PoolArenaArray [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GCMemtableMap [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=DUP_LS_SET [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 1,120 used= 1,056 count= 1 avg_used= 1,056 block_cnt= 1 chunk_cnt= 1 mod=LOG_HASH_MAP [MEMORY] hold= 1,120 used= 720 count= 5 avg_used= 144 block_cnt= 5 chunk_cnt= 1 mod=ObTxLSLogCb [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=DiskUsageTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=FlushTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TabShellTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=CheckPointTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TLD_TIMER [MEMORY] hold= 848 used= 776 count= 1 avg_used= 776 block_cnt= 1 chunk_cnt= 1 mod=HashBucPsSessIn [MEMORY] hold= 848 used= 776 count= 1 avg_used= 776 block_cnt= 1 chunk_cnt= 1 mod=DepTabletSet [MEMORY] hold= 720 used= 648 count= 1 avg_used= 648 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfo [MEMORY] hold= 624 used= 552 count= 1 avg_used= 552 block_cnt= 1 chunk_cnt= 1 mod=LSFreeze [MEMORY] hold= 576 used= 472 count= 1 avg_used= 472 block_cnt= 1 chunk_cnt= 1 mod=Election [MEMORY] hold= 528 used= 120 count= 5 avg_used= 24 block_cnt= 2 chunk_cnt= 1 mod=FreezeTask [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=HAScheduler [MEMORY] hold= 336 used= 144 count= 2 avg_used= 72 block_cnt= 2 chunk_cnt= 2 mod=TntSrvObjPool [MEMORY] hold= 192 used= 104 count= 1 avg_used= 104 block_cnt= 1 chunk_cnt= 1 mod=RedoLogBuckLock [MEMORY] hold= 192 used= 64 count= 2 avg_used= 32 block_cnt= 1 chunk_cnt= 1 mod=PalfFSCbNode [MEMORY] hold= 176 used= 112 count= 1 avg_used= 112 block_cnt= 1 chunk_cnt= 1 mod=Coordinator [MEMORY] hold= 160 used= 16 count= 2 avg_used= 8 block_cnt= 2 chunk_cnt= 2 mod=ObLogEXTHandler [MEMORY] hold= 128 used= 56 count= 1 avg_used= 56 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfoMap [MEMORY] hold= 112 used= 41 count= 1 avg_used= 41 block_cnt= 1 chunk_cnt= 1 mod=KeepAliveBuf [MEMORY] hold= 96 used= 32 count= 1 avg_used= 32 block_cnt= 1 chunk_cnt= 1 mod=RebuildCbNode [MEMORY] hold= 96 used= 32 count= 1 avg_used= 32 block_cnt= 1 chunk_cnt= 1 mod=PalfRCCbNode [MEMORY] hold= 80 used= 16 count= 1 avg_used= 16 block_cnt= 1 chunk_cnt= 1 mod=logservice [MEMORY] hold= 1,617,364,480 used= 1,582,025,881 count= 65,216 avg_used= 24,258 mod=SUMMARY [2024-06-25 17:00:18.759576] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=28] [MEMORY] tenant_id= 1 ctx_id= MEMSTORE_CTX_ID hold= 285,212,672 used= 282,984,448 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 282,984,448 used= 282,854,976 count= 136 avg_used= 2,079,816 block_cnt= 136 chunk_cnt= 136 mod=Memstore [MEMORY] hold= 282,984,448 used= 282,854,976 count= 136 avg_used= 2,079,816 mod=SUMMARY [2024-06-25 17:00:18.759584] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1 ctx_id= TRANS_CTX_MGR_ID hold= 2,097,152 used= 729,088 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 729,088 used= 722,496 count= 1 avg_used= 722,496 block_cnt= 1 chunk_cnt= 1 mod=PartTranCtxMgr [MEMORY] hold= 729,088 used= 722,496 count= 1 avg_used= 722,496 mod=SUMMARY [2024-06-25 17:00:18.759589] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.759593] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1 ctx_id= PLAN_CACHE_CTX_ID hold= 312,475,648 used= 291,707,968 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 269,003,072 used= 247,743,202 count= 10,744 avg_used= 23,058 block_cnt= 10,744 chunk_cnt= 149 mod=SqlPhyPlan [MEMORY] hold= 16,283,584 used= 15,003,888 count= 924 avg_used= 16,237 block_cnt= 924 chunk_cnt= 147 mod=SqlPlanCache [MEMORY] hold= 3,742,464 used= 3,624,768 count= 1,827 avg_used= 1,984 block_cnt= 459 chunk_cnt= 132 mod=CreateContext [MEMORY] hold= 1,440,000 used= 1,428,480 count= 180 avg_used= 7,936 block_cnt= 180 chunk_cnt= 40 mod=OB_SQLSTAT_OBJ [MEMORY] hold= 1,237,120 used= 1,196,088 count= 140 avg_used= 8,543 block_cnt= 140 chunk_cnt= 2 mod=SqlPhyPlObj [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=PlanCache [MEMORY] hold= 291,707,968 used= 268,998,026 count= 13,817 avg_used= 19,468 mod=SUMMARY [2024-06-25 17:00:18.759598] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.759603] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1 ctx_id= GLIBC hold= 4,194,304 used= 2,409,008 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 2,360,128 used= 1,878,143 count= 6,788 avg_used= 276 block_cnt= 296 chunk_cnt= 2 mod=PlJit [MEMORY] hold= 48,880 used= 45,842 count= 43 avg_used= 1,066 block_cnt= 13 chunk_cnt= 1 mod=PlCodeGen [MEMORY] hold= 2,409,008 used= 1,923,985 count= 6,831 avg_used= 281 mod=SUMMARY [2024-06-25 17:00:18.759603] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.759608] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.759611] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1 ctx_id= CO_STACK hold= 146,800,640 used= 143,990,784 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 143,990,784 used= 143,722,944 count= 279 avg_used= 515,136 block_cnt= 279 chunk_cnt= 70 mod=CoStack [MEMORY] hold= 143,990,784 used= 143,722,944 count= 279 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.759612] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.759621] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1 ctx_id= KVSTORE_CACHE_ID hold= 14,854,127,616 used= 14,854,127,616 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 14,854,127,616 used= 0 count= 7,083 avg_used= 0 block_cnt= 0 chunk_cnt= 0 mod=KvstorCacheMb [MEMORY] hold= 14,854,127,616 used= 0 count= 7,083 avg_used= 0 mod=SUMMARY [2024-06-25 17:00:18.759630] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1 ctx_id= META_OBJ_CTX_ID hold= 20,971,520 used= 18,849,792 limit= 8,079,907,220 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 13,271,040 used= 11,806,380 count= 180 avg_used= 65,591 block_cnt= 180 chunk_cnt= 8 mod=L_TabletPool [MEMORY] hold= 5,177,344 used= 5,167,232 count= 79 avg_used= 65,408 block_cnt= 79 chunk_cnt= 5 mod=N_TabletPool [MEMORY] hold= 401,408 used= 400,064 count= 2 avg_used= 200,032 block_cnt= 2 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 18,849,792 used= 17,373,676 count= 261 avg_used= 66,565 mod=SUMMARY [2024-06-25 17:00:18.759638] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1 ctx_id= TX_CALLBACK_CTX_ID hold= 20,971,520 used= 16,232,768 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 16,232,768 used= 16,102,144 count= 2,029 avg_used= 7,936 block_cnt= 2,029 chunk_cnt= 10 mod=MemtableCallbac [MEMORY] hold= 16,232,768 used= 16,102,144 count= 2,029 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.759646] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1 ctx_id= LOB_CTX_ID hold= 2,097,152 used= 65,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LobAllocator [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 mod=SUMMARY [2024-06-25 17:00:18.759652] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1 ctx_id= PS_CACHE_CTX_ID hold= 2,097,152 used= 26,624 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 26,624 used= 25,214 count= 20 avg_used= 1,260 block_cnt= 5 chunk_cnt= 1 mod=SqlPsCache [MEMORY] hold= 26,624 used= 25,214 count= 20 avg_used= 1,260 mod=SUMMARY [2024-06-25 17:00:18.759664] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1 ctx_id= TX_DATA_TABLE hold= 69,226,496 used= 64,977,280 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 47,106,816 used= 46,727,168 count= 5,888 avg_used= 7,936 block_cnt= 5,888 chunk_cnt= 25 mod=TX_DATA_SLICE [MEMORY] hold= 17,862,464 used= 17,833,792 count= 3 avg_used= 5,944,597 block_cnt= 3 chunk_cnt= 3 mod=MEMTABLE_ARENA [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=TX_DATA_ARENA [MEMORY] hold= 64,977,280 used= 64,568,896 count= 5,892 avg_used= 10,958 mod=SUMMARY [2024-06-25 17:00:18.759674] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1 ctx_id= MDS_DATA_ID hold= 2,097,152 used= 64,000 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 64,000 used= 63,488 count= 8 avg_used= 7,936 block_cnt= 8 chunk_cnt= 1 mod=MdsTable [MEMORY] hold= 64,000 used= 63,488 count= 8 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.759695] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant: 1001, limit: 1,073,741,824 hold: 343,166,976 rpc_hold: 0 cache_hold: 14,680,064 cache_used: 14,680,064 cache_item_count: 7 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 183,783,424 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PLAN_CACHE_CTX_ID hold_bytes= 48,234,496 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= GLIBC hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 88,080,384 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= META_OBJ_CTX_ID hold_bytes= 2,097,152 limit= 214,748,360 [MEMORY] ctx_id= LOB_CTX_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY][PM] tid= 63495 used= 0 hold= 2,097,152 pm=0x2b3aa75d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 63496 used= 0 hold= 4,194,304 pm=0x2b3aa7656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 63497 used= 0 hold= 4,194,304 pm=0x2b3aa76d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 63499 used= 0 hold= 2,097,152 pm=0x2b3aa77d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= summary used= 0 hold= 12,582,912 [2024-06-25 17:00:18.759796] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1001 ctx_id= DEFAULT_CTX_ID hold= 183,783,424 used= 139,026,848 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 13,303,808 used= 13,289,408 count= 8 avg_used= 1,661,176 block_cnt= 8 chunk_cnt= 8 mod=MysqlRequesReco [MEMORY] hold= 12,091,392 used= 10,796,448 count= 164 avg_used= 65,832 block_cnt= 164 chunk_cnt= 21 mod=[T]ObSessionDIB [MEMORY] hold= 10,711,040 used= 10,670,496 count= 10 avg_used= 1,067,049 block_cnt= 10 chunk_cnt= 6 mod=IoControl [MEMORY] hold= 8,895,632 used= 8,810,249 count= 79 avg_used= 111,522 block_cnt= 25 chunk_cnt= 5 mod=OmtTenant [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=ArcFetchQueue [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=RCSrv [MEMORY] hold= 8,257,536 used= 8,240,064 count= 2 avg_used= 4,120,032 block_cnt= 2 chunk_cnt= 2 mod=FreeTbltStream [MEMORY] hold= 6,561,792 used= 5,871,063 count= 89 avg_used= 65,967 block_cnt= 89 chunk_cnt= 6 mod=SqlDtlBuf [MEMORY] hold= 4,161,536 used= 4,159,488 count= 2 avg_used= 2,079,744 block_cnt= 2 chunk_cnt= 2 mod=CACHE_MAP_NODE [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMem [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 10 mod=CkptDgnMemCU [MEMORY] hold= 3,145,728 used= 2,228,224 count= 128 avg_used= 17,408 block_cnt= 128 chunk_cnt= 5 mod=SqlDtlQueue [MEMORY] hold= 2,756,608 used= 2,720,005 count= 3 avg_used= 906,668 block_cnt= 3 chunk_cnt= 2 mod=SqlDtl1stBuf [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=TransTimeWheel [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=XATimeWheel [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=MediumTabletMap [MEMORY] hold= 2,375,680 used= 2,359,536 count= 2 avg_used= 1,179,768 block_cnt= 2 chunk_cnt= 2 mod=HashBuckLCSta [MEMORY] hold= 2,248,704 used= 2,228,224 count= 1 avg_used= 2,228,224 block_cnt= 1 chunk_cnt= 1 mod=LogIOCb [MEMORY] hold= 1,777,664 used= 1,770,696 count= 1 avg_used= 1,770,696 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 1,744,896 used= 1,540,096 count= 25 avg_used= 61,603 block_cnt= 25 chunk_cnt= 8 mod=[T]char [MEMORY] hold= 1,213,760 used= 1,195,056 count= 258 avg_used= 4,632 block_cnt= 258 chunk_cnt= 23 mod=WorkerMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=RewriteRuleMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMonMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPlanCac [MEMORY] hold= 958,464 used= 950,272 count= 1 avg_used= 950,272 block_cnt= 1 chunk_cnt= 1 mod=IOWorkerLQ [MEMORY] hold= 933,888 used= 931,072 count= 1 avg_used= 931,072 block_cnt= 1 chunk_cnt= 1 mod=ArcSenderQueue [MEMORY] hold= 829,632 used= 804,864 count= 258 avg_used= 3,119 block_cnt= 130 chunk_cnt= 2 mod=LSMap [MEMORY] hold= 802,944 used= 786,432 count= 256 avg_used= 3,072 block_cnt= 128 chunk_cnt= 4 mod=TabletToLS [MEMORY] hold= 802,816 used= 800,000 count= 1 avg_used= 800,000 block_cnt= 1 chunk_cnt= 1 mod=SqlFltSpanRec [MEMORY] hold= 794,624 used= 786,536 count= 1 avg_used= 786,536 block_cnt= 1 chunk_cnt= 1 mod=TabletMap [MEMORY] hold= 786,560 used= 524,352 count= 33 avg_used= 15,889 block_cnt= 33 chunk_cnt= 1 mod=LogAlloc [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 663,552 used= 655,360 count= 1 avg_used= 655,360 block_cnt= 1 chunk_cnt= 1 mod=FetchLog [MEMORY] hold= 655,360 used= 649,216 count= 2 avg_used= 324,608 block_cnt= 2 chunk_cnt= 1 mod=ServerObjecPool [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTF [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=ElectTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=FrzTrigger [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MdsT [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DetectorTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTR [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MultiVersionGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DupTbLease [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=OBJLockGC [MEMORY] hold= 598,016 used= 590,232 count= 1 avg_used= 590,232 block_cnt= 1 chunk_cnt= 1 mod=DagNetIdMap [MEMORY] hold= 532,480 used= 531,072 count= 1 avg_used= 531,072 block_cnt= 1 chunk_cnt= 1 mod=TransService [MEMORY] hold= 523,776 used= 523,264 count= 8 avg_used= 65,408 block_cnt= 8 chunk_cnt= 6 mod=ResultSet [MEMORY] hold= 488,000 used= 484,096 count= 61 avg_used= 7,936 block_cnt= 61 chunk_cnt= 16 mod=ParamCharsType [MEMORY] hold= 434,176 used= 416,870 count= 4 avg_used= 104,217 block_cnt= 4 chunk_cnt= 2 mod=SqlMemMgr [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ApplySrv [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ReplaySrv [MEMORY] hold= 392,640 used= 383,936 count= 7 avg_used= 54,848 block_cnt= 7 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 385,024 used= 375,520 count= 2 avg_used= 187,760 block_cnt= 2 chunk_cnt= 1 mod=bf_queue [MEMORY] hold= 327,680 used= 325,376 count= 1 avg_used= 325,376 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMon [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=DmlStatHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=ColUsagHashMap [MEMORY] hold= 229,376 used= 227,744 count= 1 avg_used= 227,744 block_cnt= 1 chunk_cnt= 1 mod=SQLSessionInfo [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagMap [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagNetMap [MEMORY] hold= 200,704 used= 196,608 count= 64 avg_used= 3,072 block_cnt= 32 chunk_cnt= 4 mod=TxCtxMgr [MEMORY] hold= 147,456 used= 139,264 count= 1 avg_used= 139,264 block_cnt= 1 chunk_cnt= 1 mod=RFLTaskQueue [MEMORY] hold= 114,688 used= 106,496 count= 1 avg_used= 106,496 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlMgr [MEMORY] hold= 106,496 used= 98,320 count= 1 avg_used= 98,320 block_cnt= 1 chunk_cnt= 1 mod=T3MBucket [MEMORY] hold= 106,496 used= 98,320 count= 1 avg_used= 98,320 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 101,152 used= 100,320 count= 13 avg_used= 7,716 block_cnt= 13 chunk_cnt= 4 mod=TLDecoderAlloc [MEMORY] hold= 98,304 used= 94,848 count= 1 avg_used= 94,848 block_cnt= 1 chunk_cnt= 1 mod=ResourceGroup [MEMORY] hold= 90,112 used= 82,048 count= 1 avg_used= 82,048 block_cnt= 1 chunk_cnt= 1 mod=MetaMemMgr [MEMORY] hold= 81,920 used= 42,720 count= 5 avg_used= 8,544 block_cnt= 5 chunk_cnt= 5 mod=[T]ObDSActionAr [MEMORY] hold= 73,728 used= 72,736 count= 1 avg_used= 72,736 block_cnt= 1 chunk_cnt= 1 mod=LogSharedQueueT [MEMORY] hold= 56,784 used= 56,336 count= 7 avg_used= 8,048 block_cnt= 7 chunk_cnt= 6 mod=[T]ObPerfEventR [MEMORY] hold= 48,832 used= 44,840 count= 2 avg_used= 22,420 block_cnt= 2 chunk_cnt= 2 mod=DynamicFactor [MEMORY] hold= 45,200 used= 21,136 count= 327 avg_used= 64 block_cnt= 54 chunk_cnt= 22 mod=Coro [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=Autoincrement [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisBkt [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 5 mod=NotParamInfo [MEMORY] hold= 32,768 used= 25,600 count= 1 avg_used= 25,600 block_cnt= 1 chunk_cnt= 1 mod=TSQLSessionMgr [MEMORY] hold= 32,768 used= 17,408 count= 2 avg_used= 8,704 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTTP [MEMORY] hold= 32,768 used= 24,688 count= 2 avg_used= 12,344 block_cnt= 2 chunk_cnt= 1 mod=TLD_ClientTask [MEMORY] hold= 32,256 used= 31,808 count= 7 avg_used= 4,544 block_cnt= 7 chunk_cnt= 5 mod=[T]ObTraceEvent [MEMORY] hold= 31,488 used= 31,232 count= 4 avg_used= 7,808 block_cnt= 4 chunk_cnt= 3 mod=HashNodeLCSta [MEMORY] hold= 25,424 used= 19,232 count= 2 avg_used= 9,616 block_cnt= 2 chunk_cnt= 1 mod=HashBuckDmId [MEMORY] hold= 24,576 used= 18,456 count= 1 avg_used= 18,456 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmChe [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 21,760 used= 20,480 count= 20 avg_used= 1,024 block_cnt= 17 chunk_cnt= 11 mod=TGTimer [MEMORY] hold= 16,768 used= 8,448 count= 3 avg_used= 2,816 block_cnt= 2 chunk_cnt= 1 mod=BaseLogWriter [MEMORY] hold= 16,384 used= 8,448 count= 1 avg_used= 8,448 block_cnt= 1 chunk_cnt= 1 mod=PalfEnv [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TableCtxMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=backupTaskSched [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TabletStats [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_AssignedMgr [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TblCtxIMgr [MEMORY] hold= 15,936 used= 3,936 count= 164 avg_used= 24 block_cnt= 44 chunk_cnt= 20 mod=[T]MemoryContex [MEMORY] hold= 15,744 used= 15,616 count= 2 avg_used= 7,808 block_cnt= 2 chunk_cnt= 2 mod=HashNodePlanCac [MEMORY] hold= 12,912 used= 8,664 count= 59 avg_used= 146 block_cnt= 1 chunk_cnt= 1 mod=TLDecoderCtx [MEMORY] hold= 8,192 used= 8,056 count= 2 avg_used= 4,028 block_cnt= 2 chunk_cnt= 1 mod=StoLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LCLSender [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ShareBlksMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LSLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LSIter [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LsElectRefInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MdsDebugMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=REPLAY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=APPLY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DASIDCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ObDMMDL [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LockWaitMgr [MEMORY] hold= 7,088 used= 6,912 count= 2 avg_used= 3,456 block_cnt= 2 chunk_cnt= 2 mod=CACHE_INST [MEMORY] hold= 6,416 used= 6,216 count= 3 avg_used= 2,072 block_cnt= 2 chunk_cnt= 2 mod=DetectManager [MEMORY] hold= 5,152 used= 4,704 count= 7 avg_used= 672 block_cnt= 4 chunk_cnt= 4 mod=[T]ObWarningBuf [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResRuleIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=PxPoolBkt [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResGrpIdMap [MEMORY] hold= 4,160 used= 4,096 count= 1 avg_used= 4,096 block_cnt= 1 chunk_cnt= 1 mod=MacroFile [MEMORY] hold= 3,776 used= 3,712 count= 1 avg_used= 3,712 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlDfc [MEMORY] hold= 3,360 used= 2,912 count= 7 avg_used= 416 block_cnt= 5 chunk_cnt= 4 mod=OMT_Worker [MEMORY] hold= 2,304 used= 1,792 count= 8 avg_used= 224 block_cnt= 1 chunk_cnt= 1 mod=LogIOTask [MEMORY] hold= 1,920 used= 1,800 count= 1 avg_used= 1,800 block_cnt= 1 chunk_cnt= 1 mod=PxResMgr [MEMORY] hold= 1,920 used= 1,856 count= 1 avg_used= 1,856 block_cnt= 1 chunk_cnt= 1 mod=LogIOWS [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TabStatMgrLock [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GCMemtableMap [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=DUP_LS_SET [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 1,120 used= 1,056 count= 1 avg_used= 1,056 block_cnt= 1 chunk_cnt= 1 mod=LOG_HASH_MAP [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TLD_TIMER [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TabShellTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=CheckPointTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=DiskUsageTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=FlushTimer [MEMORY] hold= 960 used= 816 count= 2 avg_used= 408 block_cnt= 2 chunk_cnt= 2 mod=[T]ObTLDecoderC [MEMORY] hold= 768 used= 640 count= 2 avg_used= 320 block_cnt= 1 chunk_cnt= 1 mod=PoolArenaArray [MEMORY] hold= 720 used= 648 count= 1 avg_used= 648 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfo [MEMORY] hold= 624 used= 552 count= 1 avg_used= 552 block_cnt= 1 chunk_cnt= 1 mod=LSFreeze [MEMORY] hold= 480 used= 120 count= 5 avg_used= 24 block_cnt= 1 chunk_cnt= 1 mod=FreezeTask [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=Scheduler [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=HAScheduler [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=MSTXCTX [MEMORY] hold= 336 used= 144 count= 2 avg_used= 72 block_cnt= 2 chunk_cnt= 1 mod=TntSrvObjPool [MEMORY] hold= 240 used= 6 count= 3 avg_used= 2 block_cnt= 3 chunk_cnt= 3 mod=[T]TSIUseWeak [MEMORY] hold= 192 used= 128 count= 1 avg_used= 128 block_cnt= 1 chunk_cnt= 1 mod=NegParamIdex [MEMORY] hold= 176 used= 112 count= 1 avg_used= 112 block_cnt= 1 chunk_cnt= 1 mod=Coordinator [MEMORY] hold= 160 used= 16 count= 2 avg_used= 8 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTHandler [MEMORY] hold= 128 used= 56 count= 1 avg_used= 56 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfoMap [MEMORY] hold= 80 used= 16 count= 1 avg_used= 16 block_cnt= 1 chunk_cnt= 1 mod=logservice [MEMORY] hold= 139,026,848 used= 134,626,521 count= 3,835 avg_used= 35,104 mod=SUMMARY [2024-06-25 17:00:18.759829] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=20] [MEMORY] tenant_id= 1001 ctx_id= PLAN_CACHE_CTX_ID hold= 48,234,496 used= 36,767,664 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 33,025,072 used= 30,293,379 count= 1,440 avg_used= 21,037 block_cnt= 1,440 chunk_cnt= 22 mod=SqlPhyPlan [MEMORY] hold= 1,384,000 used= 1,307,704 count= 146 avg_used= 8,956 block_cnt= 146 chunk_cnt= 20 mod=SqlPlanCache [MEMORY] hold= 1,237,120 used= 1,196,088 count= 140 avg_used= 8,543 block_cnt= 140 chunk_cnt= 3 mod=SqlPhyPlObj [MEMORY] hold= 655,488 used= 634,880 count= 320 avg_used= 1,984 block_cnt= 81 chunk_cnt= 20 mod=CreateContext [MEMORY] hold= 464,256 used= 460,288 count= 58 avg_used= 7,936 block_cnt= 58 chunk_cnt= 13 mod=OB_SQLSTAT_OBJ [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=PlanCache [MEMORY] hold= 36,767,664 used= 33,893,939 count= 2,106 avg_used= 16,093 mod=SUMMARY [2024-06-25 17:00:18.759840] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1001 ctx_id= GLIBC hold= 4,194,304 used= 2,327,680 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 2,278,848 used= 1,819,125 count= 6,525 avg_used= 278 block_cnt= 288 chunk_cnt= 2 mod=PlJit [MEMORY] hold= 48,832 used= 45,810 count= 42 avg_used= 1,090 block_cnt= 13 chunk_cnt= 1 mod=PlCodeGen [MEMORY] hold= 2,327,680 used= 1,864,935 count= 6,567 avg_used= 283 mod=SUMMARY [2024-06-25 17:00:18.759848] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1001 ctx_id= CO_STACK hold= 88,080,384 used= 85,155,840 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 85,155,840 used= 84,997,440 count= 165 avg_used= 515,136 block_cnt= 165 chunk_cnt= 42 mod=CoStack [MEMORY] hold= 85,155,840 used= 84,997,440 count= 165 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.759858] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1001 ctx_id= KVSTORE_CACHE_ID hold= 14,680,064 used= 14,680,064 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 14,680,064 used= 0 count= 7 avg_used= 0 block_cnt= 0 chunk_cnt= 0 mod=KvstorCacheMb [MEMORY] hold= 14,680,064 used= 0 count= 7 avg_used= 0 mod=SUMMARY [2024-06-25 17:00:18.759855] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164325C300C3-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301544, send_timestamp=1719305996458310) [2024-06-25 17:00:18.759866] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1001 ctx_id= META_OBJ_CTX_ID hold= 2,097,152 used= 89,920 limit= 214,748,360 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 89,920 used= 84,768 count= 2 avg_used= 42,384 block_cnt= 2 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 89,920 used= 84,768 count= 2 avg_used= 42,384 mod=SUMMARY [2024-06-25 17:00:18.759876] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1001][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1001 ctx_id= LOB_CTX_ID hold= 2,097,152 used= 65,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LobAllocator [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 mod=SUMMARY [2024-06-25 17:00:18.759915] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant: 1002, limit: 1,073,741,824 hold: 933,953,536 rpc_hold: 0 cache_hold: 123,731,968 cache_used: 123,731,968 cache_item_count: 59 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 548,995,072 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MEMSTORE_CTX_ID hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= EXECUTE_CTX_ID hold_bytes= 6,291,456 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= TRANS_CTX_MGR_ID hold_bytes= 3,694,592 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PLAN_CACHE_CTX_ID hold_bytes= 47,325,184 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 102,760,448 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= META_OBJ_CTX_ID hold_bytes= 33,341,440 limit= 214,748,360 [MEMORY] ctx_id= TX_CALLBACK_CTX_ID hold_bytes= 5,046,272 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= LOB_CTX_ID hold_bytes= 81,920 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= RPC_CTX_ID hold_bytes= 10,485,760 limit= 1,073,741,800 [MEMORY] ctx_id= TX_DATA_TABLE hold_bytes= 45,776,896 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MDS_DATA_ID hold_bytes= 2,228,224 limit= 9,223,372,036,854,775,807 [MEMORY][PM] tid= 63321 used= 0 hold= 2,097,152 pm=0x2b3a983d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 63325 used= 0 hold= 2,097,152 pm=0x2b3a987d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 65259 used= 0 hold= 4,194,304 pm=0x2b3abc1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 12536 used= 197,720 hold= 4,194,304 pm=0x2bcfc9bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 12807 used= 131,136 hold= 4,194,304 pm=0x2b3c17152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 27554 used= 336,024 hold= 4,194,304 pm=0x2bd048ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= summary used= 664,880 hold= 20,971,520 [2024-06-25 17:00:18.760017] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=3] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1003) [2024-06-25 17:00:18.760028] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=9] start lib cache evict(tenant_id=1003, mem_hold=230686720, mem_limit=4026531840, cache_obj_num=717, cache_node_num=717) [2024-06-25 17:00:18.760036] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=7] end lib cache evict(tenant_id=1003, cache_evict_num=0, mem_hold=230686720, mem_limit=4026531840, cache_obj_num=717, cache_node_num=717) [2024-06-25 17:00:18.760041] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=4] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:18.760060] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=6] [MEMORY] tenant_id= 1002 ctx_id= DEFAULT_CTX_ID hold= 548,995,072 used= 470,974,960 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 91 washed_blocks= 767 washed_size= 92,307,456 [MEMORY] hold= 209,817,600 used= 209,715,200 count= 5 avg_used= 41,943,040 block_cnt= 5 chunk_cnt= 5 mod=LogGroupBuffer [MEMORY] hold= 22,888,448 used= 22,877,184 count= 11 avg_used= 2,079,744 block_cnt= 11 chunk_cnt= 11 mod=CACHE_MAP_NODE [MEMORY] hold= 18,653,184 used= 16,689,651 count= 253 avg_used= 65,967 block_cnt= 253 chunk_cnt= 64 mod=SqlDtlBuf [MEMORY] hold= 18,616,320 used= 18,493,440 count= 10 avg_used= 1,849,344 block_cnt= 10 chunk_cnt= 8 mod=LogDIOAligned [MEMORY] hold= 15,777,792 used= 14,024,704 count= 214 avg_used= 65,536 block_cnt= 214 chunk_cnt= 51 mod=MinorMergeMgr [MEMORY] hold= 14,008,320 used= 12,508,080 count= 190 avg_used= 65,832 block_cnt= 190 chunk_cnt= 30 mod=[T]ObSessionDIB [MEMORY] hold= 13,303,808 used= 13,289,408 count= 8 avg_used= 1,661,176 block_cnt= 8 chunk_cnt= 8 mod=MysqlRequesReco [MEMORY] hold= 10,776,576 used= 10,735,904 count= 11 avg_used= 975,991 block_cnt= 11 chunk_cnt= 7 mod=IoControl [MEMORY] hold= 8,895,632 used= 8,810,249 count= 79 avg_used= 111,522 block_cnt= 25 chunk_cnt= 5 mod=OmtTenant [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=RCSrv [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=ArcFetchQueue [MEMORY] hold= 8,257,536 used= 8,249,344 count= 8 avg_used= 1,031,168 block_cnt= 8 chunk_cnt= 6 mod=Compressor [MEMORY] hold= 8,257,536 used= 8,240,064 count= 2 avg_used= 4,120,032 block_cnt= 2 chunk_cnt= 2 mod=FreeTbltStream [MEMORY] hold= 5,529,600 used= 4,915,200 count= 75 avg_used= 65,536 block_cnt= 75 chunk_cnt= 36 mod=MajorMergeMgr [MEMORY] hold= 5,120,000 used= 5,079,040 count= 5 avg_used= 1,015,808 block_cnt= 5 chunk_cnt= 3 mod=CommitVersions [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 10 mod=CkptDgnMemCU [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMem [MEMORY] hold= 3,680,096 used= 3,649,568 count= 471 avg_used= 7,748 block_cnt= 471 chunk_cnt= 79 mod=TLDecoderAlloc [MEMORY] hold= 3,416,064 used= 3,014,656 count= 49 avg_used= 61,523 block_cnt= 49 chunk_cnt= 21 mod=[T]char [MEMORY] hold= 3,145,728 used= 2,228,224 count= 128 avg_used= 17,408 block_cnt= 128 chunk_cnt= 6 mod=SqlDtlQueue [MEMORY] hold= 2,756,608 used= 2,720,005 count= 3 avg_used= 906,668 block_cnt= 3 chunk_cnt= 2 mod=SqlDtl1stBuf [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=XATimeWheel [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=TransTimeWheel [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 2,523,136 used= 2,505,184 count= 11 avg_used= 227,744 block_cnt= 11 chunk_cnt= 10 mod=SQLSessionInfo [MEMORY] hold= 2,486,272 used= 2,457,920 count= 2 avg_used= 1,228,960 block_cnt= 2 chunk_cnt= 2 mod=HashBuckDmId [MEMORY] hold= 2,437,120 used= 2,414,336 count= 2 avg_used= 1,207,168 block_cnt= 2 chunk_cnt= 2 mod=SqlPlanMon [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=MediumTabletMap [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmChe [MEMORY] hold= 2,375,680 used= 2,359,536 count= 2 avg_used= 1,179,768 block_cnt= 2 chunk_cnt= 2 mod=HashBuckLCSta [MEMORY] hold= 2,248,704 used= 2,228,224 count= 1 avg_used= 2,228,224 block_cnt= 1 chunk_cnt= 1 mod=LogIOCb [MEMORY] hold= 2,080,768 used= 2,079,744 count= 1 avg_used= 2,079,744 block_cnt= 1 chunk_cnt= 1 mod=LogReplayTask [MEMORY] hold= 1,925,120 used= 1,885,600 count= 5 avg_used= 377,120 block_cnt= 5 chunk_cnt= 2 mod=ClogGe [MEMORY] hold= 1,784,704 used= 1,746,772 count= 14 avg_used= 124,769 block_cnt= 14 chunk_cnt= 14 mod=CsSstableReader [MEMORY] hold= 1,777,664 used= 1,770,696 count= 1 avg_used= 1,770,696 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 1,720,320 used= 1,541,568 count= 42 avg_used= 36,704 block_cnt= 42 chunk_cnt= 21 mod=CommSysVarFac [MEMORY] hold= 1,507,328 used= 1,504,384 count= 23 avg_used= 65,408 block_cnt= 23 chunk_cnt= 15 mod=MemTblObj [MEMORY] hold= 1,376,256 used= 1,343,488 count= 4 avg_used= 335,872 block_cnt= 4 chunk_cnt= 1 mod=IOWorkerLQ [MEMORY] hold= 1,265,328 used= 1,253,464 count= 60 avg_used= 20,891 block_cnt= 60 chunk_cnt= 26 mod=TabletMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPlanCac [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMonMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=RewriteRuleMap [MEMORY] hold= 1,045,344 used= 710,016 count= 4,602 avg_used= 154 block_cnt= 17 chunk_cnt= 14 mod=TLDecoderCtx [MEMORY] hold= 957,088 used= 854,886 count= 42 avg_used= 20,354 block_cnt= 39 chunk_cnt= 26 mod=SqlSessiQuerSql [MEMORY] hold= 933,888 used= 931,072 count= 1 avg_used= 931,072 block_cnt= 1 chunk_cnt= 1 mod=ArcSenderQueue [MEMORY] hold= 884,736 used= 461,376 count= 54 avg_used= 8,544 block_cnt= 54 chunk_cnt= 9 mod=[T]ObDSActionAr [MEMORY] hold= 829,632 used= 804,864 count= 258 avg_used= 3,119 block_cnt= 130 chunk_cnt= 2 mod=LSMap [MEMORY] hold= 811,008 used= 720,896 count= 11 avg_used= 65,536 block_cnt= 11 chunk_cnt= 7 mod=SuspectInfoMgr [MEMORY] hold= 802,944 used= 786,432 count= 256 avg_used= 3,072 block_cnt= 128 chunk_cnt= 4 mod=TabletToLS [MEMORY] hold= 802,816 used= 800,000 count= 1 avg_used= 800,000 block_cnt= 1 chunk_cnt= 1 mod=SqlFltSpanRec [MEMORY] hold= 786,752 used= 524,352 count= 36 avg_used= 14,565 block_cnt= 34 chunk_cnt= 2 mod=LogAlloc [MEMORY] hold= 738,592 used= 732,448 count= 94 avg_used= 7,792 block_cnt= 94 chunk_cnt= 12 mod=CkptDgnMemNode [MEMORY] hold= 712,000 used= 706,304 count= 89 avg_used= 7,936 block_cnt= 89 chunk_cnt= 21 mod=SqlSession [MEMORY] hold= 663,552 used= 655,360 count= 1 avg_used= 655,360 block_cnt= 1 chunk_cnt= 1 mod=FetchLog [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 655,360 used= 649,216 count= 2 avg_used= 324,608 block_cnt= 2 chunk_cnt= 1 mod=ServerObjecPool [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DupTbLease [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTF [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTR [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DetectorTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MultiVersionGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=OBJLockGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=FrzTrigger [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MdsT [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=ElectTimer [MEMORY] hold= 614,400 used= 553,040 count= 10 avg_used= 55,304 block_cnt= 10 chunk_cnt= 4 mod=TabletIDSetBkt [MEMORY] hold= 606,208 used= 589,872 count= 2 avg_used= 294,936 block_cnt= 2 chunk_cnt= 1 mod=TmpBlockMap [MEMORY] hold= 598,016 used= 590,232 count= 1 avg_used= 590,232 block_cnt= 1 chunk_cnt= 1 mod=DagNetIdMap [MEMORY] hold= 589,824 used= 569,088 count= 6 avg_used= 94,848 block_cnt= 6 chunk_cnt= 5 mod=ResourceGroup [MEMORY] hold= 535,872 used= 527,424 count= 118 avg_used= 4,469 block_cnt= 112 chunk_cnt= 43 mod=WorkerMap [MEMORY] hold= 532,480 used= 531,072 count= 1 avg_used= 531,072 block_cnt= 1 chunk_cnt= 1 mod=TransService [MEMORY] hold= 523,776 used= 523,264 count= 8 avg_used= 65,408 block_cnt= 8 chunk_cnt= 7 mod=ResultSet [MEMORY] hold= 458,752 used= 457,856 count= 7 avg_used= 65,408 block_cnt= 7 chunk_cnt= 6 mod=MemTblMgrObj [MEMORY] hold= 450,560 used= 440,928 count= 3 avg_used= 146,976 block_cnt= 3 chunk_cnt= 3 mod=bf_queue [MEMORY] hold= 450,560 used= 422,240 count= 5 avg_used= 84,448 block_cnt= 5 chunk_cnt= 3 mod=LSSvr [MEMORY] hold= 441,952 used= 424,582 count= 5 avg_used= 84,916 block_cnt= 5 chunk_cnt= 4 mod=SqlMemMgr [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ApplySrv [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ReplaySrv [MEMORY] hold= 409,344 used= 399,744 count= 6 avg_used= 66,624 block_cnt= 6 chunk_cnt= 3 mod=SqlExecutor [MEMORY] hold= 403,200 used= 396,800 count= 100 avg_used= 3,968 block_cnt= 56 chunk_cnt= 14 mod=CkpDgn [MEMORY] hold= 392,640 used= 383,936 count= 7 avg_used= 54,848 block_cnt= 7 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 368,640 used= 344,064 count= 4 avg_used= 86,016 block_cnt= 4 chunk_cnt= 4 mod=DiskIteratorSto [MEMORY] hold= 336,000 used= 333,312 count= 42 avg_used= 7,936 block_cnt= 42 chunk_cnt= 20 mod=SqlSessiVarMap [MEMORY] hold= 328,000 used= 325,376 count= 41 avg_used= 7,936 block_cnt= 41 chunk_cnt= 15 mod=PlanVaIdx [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=DmlStatHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=ColUsagHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=TmpMap [MEMORY] hold= 262,144 used= 261,632 count= 4 avg_used= 65,408 block_cnt= 4 chunk_cnt= 4 mod=TmpPageCache [MEMORY] hold= 256,016 used= 253,952 count= 32 avg_used= 7,936 block_cnt= 32 chunk_cnt= 17 mod=QueryAllocator [MEMORY] hold= 245,760 used= 243,840 count= 5 avg_used= 48,768 block_cnt= 5 chunk_cnt= 3 mod=palf_env [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagMap [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagNetMap [MEMORY] hold= 200,704 used= 196,608 count= 64 avg_used= 3,072 block_cnt= 32 chunk_cnt= 4 mod=TxCtxMgr [MEMORY] hold= 196,608 used= 196,224 count= 3 avg_used= 65,408 block_cnt= 3 chunk_cnt= 2 mod=LockMemObj [MEMORY] hold= 165,680 used= 160,392 count= 20 avg_used= 8,019 block_cnt= 20 chunk_cnt= 12 mod=TabletStats [MEMORY] hold= 147,456 used= 139,264 count= 1 avg_used= 139,264 block_cnt= 1 chunk_cnt= 1 mod=RFLTaskQueue [MEMORY] hold= 124,992 used= 123,008 count= 31 avg_used= 3,968 block_cnt= 23 chunk_cnt= 10 mod=TinyAllocator [MEMORY] hold= 114,688 used= 106,496 count= 1 avg_used= 106,496 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlMgr [MEMORY] hold= 112,000 used= 111,104 count= 14 avg_used= 7,936 block_cnt= 14 chunk_cnt= 10 mod=ParamCharsType [MEMORY] hold= 106,496 used= 98,320 count= 1 avg_used= 98,320 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 106,496 used= 98,320 count= 1 avg_used= 98,320 block_cnt= 1 chunk_cnt= 1 mod=T3MBucket [MEMORY] hold= 102,336 used= 101,504 count= 13 avg_used= 7,808 block_cnt= 13 chunk_cnt= 9 mod=HashNodeLCSta [MEMORY] hold= 94,272 used= 93,504 count= 12 avg_used= 7,792 block_cnt= 12 chunk_cnt= 3 mod=CkptDgnCUNode [MEMORY] hold= 90,112 used= 82,048 count= 1 avg_used= 82,048 block_cnt= 1 chunk_cnt= 1 mod=MetaMemMgr [MEMORY] hold= 89,232 used= 88,528 count= 11 avg_used= 8,048 block_cnt= 11 chunk_cnt= 10 mod=[T]ObPerfEventR [MEMORY] hold= 88,128 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 7 mod=LSLocCache [MEMORY] hold= 81,920 used= 51,240 count= 5 avg_used= 10,248 block_cnt= 5 chunk_cnt= 3 mod=TabletSvrBucket [MEMORY] hold= 81,920 used= 68,480 count= 5 avg_used= 13,696 block_cnt= 5 chunk_cnt= 2 mod=LogReplayStatus [MEMORY] hold= 81,920 used= 57,920 count= 5 avg_used= 11,584 block_cnt= 5 chunk_cnt= 3 mod=LogApplyStatus [MEMORY] hold= 78,720 used= 78,080 count= 10 avg_used= 7,808 block_cnt= 10 chunk_cnt= 4 mod=TabletIDSetNode [MEMORY] hold= 73,728 used= 72,736 count= 1 avg_used= 72,736 block_cnt= 1 chunk_cnt= 1 mod=LogSharedQueueT [MEMORY] hold= 73,728 used= 66,176 count= 1 avg_used= 66,176 block_cnt= 1 chunk_cnt= 1 mod=Rpc [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=FlushLog [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxDataMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TmpBlockManager [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=StorMetaCacheIO [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=FlushMeta [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxCtxMemObj [MEMORY] hold= 59,904 used= 59,072 count= 13 avg_used= 4,544 block_cnt= 13 chunk_cnt= 11 mod=[T]ObTraceEvent [MEMORY] hold= 51,632 used= 24,328 count= 373 avg_used= 65 block_cnt= 89 chunk_cnt= 30 mod=Coro [MEMORY] hold= 48,832 used= 44,840 count= 2 avg_used= 22,420 block_cnt= 2 chunk_cnt= 2 mod=DynamicFactor [MEMORY] hold= 47,232 used= 46,848 count= 6 avg_used= 7,808 block_cnt= 6 chunk_cnt= 5 mod=HashNodePlanCac [MEMORY] hold= 47,232 used= 46,848 count= 6 avg_used= 7,808 block_cnt= 6 chunk_cnt= 5 mod=SuspectInfoNode [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=Autoincrement [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 1 mod=ObDMMDL [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 3 mod=MatchLsnMap [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 3 mod=MdsTableMgr [MEMORY] hold= 38,752 used= 38,016 count= 11 avg_used= 3,456 block_cnt= 11 chunk_cnt= 10 mod=CACHE_INST [MEMORY] hold= 32,768 used= 17,408 count= 2 avg_used= 8,704 block_cnt= 2 chunk_cnt= 2 mod=ObLogEXTTP [MEMORY] hold= 32,768 used= 24,688 count= 2 avg_used= 12,344 block_cnt= 2 chunk_cnt= 2 mod=TLD_ClientTask [MEMORY] hold= 32,768 used= 25,600 count= 1 avg_used= 25,600 block_cnt= 1 chunk_cnt= 1 mod=TSQLSessionMgr [MEMORY] hold= 29,760 used= 25,296 count= 62 avg_used= 408 block_cnt= 58 chunk_cnt= 27 mod=[T]ObTLDecoderC [MEMORY] hold= 25,024 used= 23,552 count= 23 avg_used= 1,024 block_cnt= 19 chunk_cnt= 12 mod=TGTimer [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 23,520 used= 23,160 count= 5 avg_used= 4,632 block_cnt= 5 chunk_cnt= 3 mod=TTLTaskBucket [MEMORY] hold= 21,008 used= 17,808 count= 42 avg_used= 424 block_cnt= 30 chunk_cnt= 16 mod=PackStateMap [MEMORY] hold= 20,880 used= 17,808 count= 42 avg_used= 424 block_cnt= 34 chunk_cnt= 18 mod=ContextsMap [MEMORY] hold= 20,880 used= 17,808 count= 42 avg_used= 424 block_cnt= 34 chunk_cnt= 18 mod=SequenceIdMap [MEMORY] hold= 20,864 used= 17,808 count= 42 avg_used= 424 block_cnt= 32 chunk_cnt= 17 mod=SequenceMap [MEMORY] hold= 18,656 used= 4,560 count= 190 avg_used= 24 block_cnt= 51 chunk_cnt= 26 mod=[T]MemoryContex [MEMORY] hold= 16,768 used= 8,448 count= 3 avg_used= 2,816 block_cnt= 2 chunk_cnt= 1 mod=BaseLogWriter [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_AssignedMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TblCtxIMgr [MEMORY] hold= 16,384 used= 8,448 count= 1 avg_used= 8,448 block_cnt= 1 chunk_cnt= 1 mod=PalfEnv [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TableCtxMgr [MEMORY] hold= 16,384 used= 10,248 count= 1 avg_used= 10,248 block_cnt= 1 chunk_cnt= 1 mod=TmpMemBlkMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=backupTaskSched [MEMORY] hold= 16,000 used= 15,872 count= 2 avg_used= 7,936 block_cnt= 2 chunk_cnt= 2 mod=ShareBlksMap [MEMORY] hold= 16,000 used= 15,872 count= 2 avg_used= 7,936 block_cnt= 2 chunk_cnt= 2 mod=LSIter [MEMORY] hold= 15,680 used= 15,360 count= 5 avg_used= 3,072 block_cnt= 5 chunk_cnt= 3 mod=LockMemtableMgr [MEMORY] hold= 10,240 used= 10,112 count= 2 avg_used= 5,056 block_cnt= 1 chunk_cnt= 1 mod=Scheduler [MEMORY] hold= 9,392 used= 9,256 count= 2 avg_used= 4,628 block_cnt= 2 chunk_cnt= 2 mod=StoLocCache [MEMORY] hold= 8,832 used= 8,064 count= 12 avg_used= 672 block_cnt= 10 chunk_cnt= 10 mod=[T]ObWarningBuf [MEMORY] hold= 8,640 used= 7,488 count= 18 avg_used= 416 block_cnt= 12 chunk_cnt= 7 mod=OMT_Worker [MEMORY] hold= 8,080 used= 7,720 count= 5 avg_used= 1,544 block_cnt= 5 chunk_cnt= 1 mod=TXBFRetryId [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LockWaitMgr [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LCLSender [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ls_id [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DASIDCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LsElectRefInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=APPLY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=REPLAY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MdsDebugMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=FetchLogTask [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=PurgeThrottle [MEMORY] hold= 7,872 used= 7,808 count= 1 avg_used= 7,808 block_cnt= 1 chunk_cnt= 1 mod=DirToBlk [MEMORY] hold= 7,856 used= 7,792 count= 1 avg_used= 7,792 block_cnt= 1 chunk_cnt= 1 mod=ColUsagNode [MEMORY] hold= 7,488 used= 7,424 count= 1 avg_used= 7,424 block_cnt= 1 chunk_cnt= 1 mod=LogIOWS [MEMORY] hold= 6,416 used= 6,216 count= 3 avg_used= 2,072 block_cnt= 2 chunk_cnt= 2 mod=DetectManager [MEMORY] hold= 5,456 used= 3,744 count= 26 avg_used= 144 block_cnt= 22 chunk_cnt= 8 mod=ObTxLSLogCb [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResRuleIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResGrpIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=PxPoolBkt [MEMORY] hold= 4,240 used= 3,880 count= 5 avg_used= 776 block_cnt= 4 chunk_cnt= 2 mod=DepTabletSet [MEMORY] hold= 4,160 used= 4,096 count= 1 avg_used= 4,096 block_cnt= 1 chunk_cnt= 1 mod=MacroFile [MEMORY] hold= 3,776 used= 3,712 count= 1 avg_used= 3,712 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlDfc [MEMORY] hold= 3,088 used= 3,016 count= 1 avg_used= 3,016 block_cnt= 1 chunk_cnt= 1 mod=HeapVar [MEMORY] hold= 2,768 used= 2,360 count= 5 avg_used= 472 block_cnt= 5 chunk_cnt= 4 mod=Election [MEMORY] hold= 2,352 used= 1,792 count= 8 avg_used= 224 block_cnt= 2 chunk_cnt= 1 mod=LogIOTask [MEMORY] hold= 2,272 used= 1,664 count= 8 avg_used= 208 block_cnt= 7 chunk_cnt= 7 mod=[T]ObMacroBlock [MEMORY] hold= 2,048 used= 1,984 count= 1 avg_used= 1,984 block_cnt= 1 chunk_cnt= 1 mod=CreateContext [MEMORY] hold= 1,920 used= 1,800 count= 1 avg_used= 1,800 block_cnt= 1 chunk_cnt= 1 mod=PxResMgr [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=MSTXCTX [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TabStatMgrLock [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=DUP_LS_SET [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GCMemtableMap [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 1,120 used= 1,056 count= 1 avg_used= 1,056 block_cnt= 1 chunk_cnt= 1 mod=LOG_HASH_MAP [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TLD_TIMER [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=CheckPointTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=FlushTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=DiskUsageTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TabShellTimer [MEMORY] hold= 1,072 used= 320 count= 10 avg_used= 32 block_cnt= 10 chunk_cnt= 6 mod=PalfFSCbNode [MEMORY] hold= 928 used= 520 count= 5 avg_used= 104 block_cnt= 5 chunk_cnt= 4 mod=RedoLogBuckLock [MEMORY] hold= 768 used= 640 count= 2 avg_used= 320 block_cnt= 1 chunk_cnt= 1 mod=PoolArenaArray [MEMORY] hold= 752 used= 648 count= 1 avg_used= 648 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfo [MEMORY] hold= 640 used= 160 count= 5 avg_used= 32 block_cnt= 5 chunk_cnt= 5 mod=RebuildCbNode [MEMORY] hold= 640 used= 16 count= 8 avg_used= 2 block_cnt= 8 chunk_cnt= 8 mod=[T]TSIUseWeak [MEMORY] hold= 624 used= 552 count= 1 avg_used= 552 block_cnt= 1 chunk_cnt= 1 mod=LSFreeze [MEMORY] hold= 592 used= 160 count= 5 avg_used= 32 block_cnt= 5 chunk_cnt= 5 mod=PalfRCCbNode [MEMORY] hold= 560 used= 205 count= 5 avg_used= 41 block_cnt= 5 chunk_cnt= 5 mod=KeepAliveBuf [MEMORY] hold= 480 used= 120 count= 5 avg_used= 24 block_cnt= 1 chunk_cnt= 1 mod=FreezeTask [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=HAScheduler [MEMORY] hold= 336 used= 144 count= 2 avg_used= 72 block_cnt= 2 chunk_cnt= 1 mod=TntSrvObjPool [MEMORY] hold= 176 used= 112 count= 1 avg_used= 112 block_cnt= 1 chunk_cnt= 1 mod=Coordinator [MEMORY] hold= 160 used= 93 count= 1 avg_used= 93 block_cnt= 1 chunk_cnt= 1 mod=TxLSLogBuf [MEMORY] hold= 160 used= 16 count= 2 avg_used= 8 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTHandler [MEMORY] hold= 128 used= 56 count= 1 avg_used= 56 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfoMap [MEMORY] hold= 80 used= 16 count= 1 avg_used= 16 block_cnt= 1 chunk_cnt= 1 mod=logservice [MEMORY] hold= 470,974,960 used= 460,645,803 count= 10,481 avg_used= 43,950 mod=SUMMARY [2024-06-25 17:00:18.760092] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=27] [MEMORY] tenant_id= 1002 ctx_id= MEMSTORE_CTX_ID hold= 4,194,304 used= 4,161,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 4,161,536 used= 4,159,632 count= 2 avg_used= 2,079,816 block_cnt= 2 chunk_cnt= 2 mod=Memstore [MEMORY] hold= 4,161,536 used= 4,159,632 count= 2 avg_used= 2,079,816 mod=SUMMARY [2024-06-25 17:00:18.760098] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1002 ctx_id= EXECUTE_CTX_ID hold= 6,291,456 used= 40,000 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 3 mod=SqlExecContext [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.760104] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1002 ctx_id= TRANS_CTX_MGR_ID hold= 3,694,592 used= 3,645,440 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 3 washed_blocks= 3 washed_size= 2,596,864 [MEMORY] hold= 3,645,440 used= 3,612,480 count= 5 avg_used= 722,496 block_cnt= 5 chunk_cnt= 3 mod=PartTranCtxMgr [MEMORY] hold= 3,645,440 used= 3,612,480 count= 5 avg_used= 722,496 mod=SUMMARY [2024-06-25 17:00:18.760114] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1002 ctx_id= PLAN_CACHE_CTX_ID hold= 47,325,184 used= 17,449,728 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 22 washed_blocks= 101 washed_size= 32,366,592 [MEMORY] hold= 13,820,672 used= 12,671,679 count= 579 avg_used= 21,885 block_cnt= 579 chunk_cnt= 36 mod=SqlPhyPlan [MEMORY] hold= 2,232,128 used= 2,214,144 count= 279 avg_used= 7,936 block_cnt= 279 chunk_cnt= 31 mod=OB_SQLSTAT_OBJ [MEMORY] hold= 798,848 used= 773,760 count= 390 avg_used= 1,984 block_cnt= 149 chunk_cnt= 31 mod=CreateContext [MEMORY] hold= 596,352 used= 557,416 count= 60 avg_used= 9,290 block_cnt= 60 chunk_cnt= 27 mod=SqlPlanCache [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=PlanCache [MEMORY] hold= 17,449,728 used= 16,218,599 count= 1,310 avg_used= 12,380 mod=SUMMARY [2024-06-25 17:00:18.760126] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1002 ctx_id= CO_STACK hold= 102,760,448 used= 98,058,240 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 98,058,240 used= 97,875,840 count= 190 avg_used= 515,136 block_cnt= 190 chunk_cnt= 49 mod=CoStack [MEMORY] hold= 98,058,240 used= 97,875,840 count= 190 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.760137] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1002 ctx_id= KVSTORE_CACHE_ID hold= 123,731,968 used= 123,731,968 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 123,731,968 used= 0 count= 59 avg_used= 0 block_cnt= 0 chunk_cnt= 0 mod=KvstorCacheMb [MEMORY] hold= 123,731,968 used= 0 count= 59 avg_used= 0 mod=SUMMARY [2024-06-25 17:00:18.760139] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABF4538-00061669548FCBE8-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301543, send_timestamp=1719305996458595) [2024-06-25 17:00:18.760146] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1002 ctx_id= META_OBJ_CTX_ID hold= 33,341,440 used= 33,062,720 limit= 214,748,360 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 17 washed_blocks= 17 washed_size= 2,310,144 [MEMORY] hold= 29,491,200 used= 26,236,400 count= 400 avg_used= 65,591 block_cnt= 400 chunk_cnt= 16 mod=L_TabletPool [MEMORY] hold= 3,481,600 used= 3,466,624 count= 53 avg_used= 65,408 block_cnt= 53 chunk_cnt= 4 mod=N_TabletPool [MEMORY] hold= 89,920 used= 84,768 count= 2 avg_used= 42,384 block_cnt= 2 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 33,062,720 used= 29,787,792 count= 455 avg_used= 65,467 mod=SUMMARY [2024-06-25 17:00:18.760155] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=6] [MEMORY] tenant_id= 1002 ctx_id= TX_CALLBACK_CTX_ID hold= 5,046,272 used= 784,128 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 3 washed_blocks= 5 washed_size= 5,439,488 [MEMORY] hold= 784,128 used= 777,728 count= 98 avg_used= 7,936 block_cnt= 98 chunk_cnt= 5 mod=MemtableCallbac [MEMORY] hold= 784,128 used= 777,728 count= 98 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.760162] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1002 ctx_id= LOB_CTX_ID hold= 81,920 used= 65,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 1 washed_blocks= 1 washed_size= 2,015,232 [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LobAllocator [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 mod=SUMMARY [2024-06-25 17:00:18.760173] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1002 ctx_id= RPC_CTX_ID hold= 10,485,760 used= 425,984 limit= 1,073,741,800 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 294,912 used= 292,608 count= 18 avg_used= 16,256 block_cnt= 18 chunk_cnt= 1 mod=OB_WRS_CLUSTER_ [MEMORY] hold= 81,920 used= 81,280 count= 5 avg_used= 16,256 block_cnt= 5 chunk_cnt= 4 mod=OB_REMOTE_EXECU [MEMORY] hold= 32,768 used= 32,512 count= 2 avg_used= 16,256 block_cnt= 2 chunk_cnt= 1 mod=[L]OB_REMOTE_EX [MEMORY] hold= 16,384 used= 16,256 count= 1 avg_used= 16,256 block_cnt= 1 chunk_cnt= 1 mod=OB_REMOTE_SYNC_ [MEMORY] hold= 425,984 used= 422,656 count= 26 avg_used= 16,256 mod=SUMMARY [2024-06-25 17:00:18.760184] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1002 ctx_id= TX_DATA_TABLE hold= 45,776,896 used= 11,951,680 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 19 washed_blocks= 47 washed_size= 25,526,272 [MEMORY] hold= 10,607,680 used= 10,525,760 count= 15 avg_used= 701,717 block_cnt= 15 chunk_cnt= 11 mod=MEMTABLE_ARENA [MEMORY] hold= 1,304,000 used= 1,293,568 count= 163 avg_used= 7,936 block_cnt= 163 chunk_cnt= 25 mod=TX_DATA_SLICE [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 1 mod=TX_DATA_ARENA [MEMORY] hold= 11,951,680 used= 11,859,008 count= 183 avg_used= 64,803 mod=SUMMARY [2024-06-25 17:00:18.760194] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1002][Y0-0000000000000000-0-0] [lt=6] [MEMORY] tenant_id= 1002 ctx_id= MDS_DATA_ID hold= 2,228,224 used= 56,000 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 1 washed_blocks= 3 washed_size= 1,966,080 [MEMORY] hold= 56,000 used= 55,552 count= 7 avg_used= 7,936 block_cnt= 7 chunk_cnt= 2 mod=MdsTable [MEMORY] hold= 56,000 used= 55,552 count= 7 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.760217] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant: 1003, limit: 80,530,636,800 hold: 845,557,760 rpc_hold: 0 cache_hold: 12,582,912 cache_used: 12,582,912 cache_item_count: 6 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 461,778,944 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PLAN_CACHE_CTX_ID hold_bytes= 230,686,720 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= GLIBC hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 132,120,576 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= META_OBJ_CTX_ID hold_bytes= 2,097,152 limit= 16,106,127,360 [MEMORY] ctx_id= LOB_CTX_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY][PM] tid= 100566 used= 0 hold= 4,194,304 pm=0x2b3b55352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100567 used= 0 hold= 4,194,304 pm=0x2b3b553d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100574 used= 0 hold= 4,194,304 pm=0x2b3b55752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100575 used= 0 hold= 4,194,304 pm=0x2b3b557d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100576 used= 0 hold= 4,194,304 pm=0x2b3b55856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= summary used= 0 hold= 20,971,520 [2024-06-25 17:00:18.760319] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1003 ctx_id= DEFAULT_CTX_ID hold= 461,778,944 used= 395,343,200 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 196,411,392 used= 196,306,880 count= 96 avg_used= 2,044,863 block_cnt= 96 chunk_cnt= 96 mod=MysqlRequesReco [MEMORY] hold= 26,009,600 used= 25,968,640 count= 2 avg_used= 12,984,320 block_cnt= 2 chunk_cnt= 2 mod=ServerObjecPool [MEMORY] hold= 18,284,544 used= 16,326,336 count= 248 avg_used= 65,832 block_cnt= 248 chunk_cnt= 24 mod=[T]ObSessionDIB [MEMORY] hold= 12,603,392 used= 12,582,952 count= 1 avg_used= 12,582,952 block_cnt= 1 chunk_cnt= 1 mod=TabletMap [MEMORY] hold= 10,711,040 used= 10,670,496 count= 10 avg_used= 1,067,049 block_cnt= 10 chunk_cnt= 6 mod=IoControl [MEMORY] hold= 8,895,632 used= 8,810,249 count= 79 avg_used= 111,522 block_cnt= 25 chunk_cnt= 5 mod=OmtTenant [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=ArcFetchQueue [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=RCSrv [MEMORY] hold= 8,257,536 used= 8,240,064 count= 2 avg_used= 4,120,032 block_cnt= 2 chunk_cnt= 2 mod=FreeTbltStream [MEMORY] hold= 8,151,040 used= 8,131,072 count= 1 avg_used= 8,131,072 block_cnt= 1 chunk_cnt= 1 mod=TransService [MEMORY] hold= 6,463,808 used= 6,364,368 count= 1,374 avg_used= 4,632 block_cnt= 1,374 chunk_cnt= 36 mod=WorkerMap [MEMORY] hold= 6,340,608 used= 5,673,162 count= 86 avg_used= 65,967 block_cnt= 86 chunk_cnt= 5 mod=SqlDtlBuf [MEMORY] hold= 4,739,072 used= 4,718,712 count= 1 avg_used= 4,718,712 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMonMap [MEMORY] hold= 4,161,536 used= 4,159,488 count= 2 avg_used= 2,079,744 block_cnt= 2 chunk_cnt= 2 mod=CACHE_MAP_NODE [MEMORY] hold= 4,055,040 used= 3,588,096 count= 57 avg_used= 62,949 block_cnt= 57 chunk_cnt= 14 mod=[T]char [MEMORY] hold= 3,944,768 used= 3,912,448 count= 491 avg_used= 7,968 block_cnt= 491 chunk_cnt= 29 mod=ParamCharsType [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMem [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMemCU [MEMORY] hold= 3,145,728 used= 2,228,224 count= 128 avg_used= 17,408 block_cnt= 128 chunk_cnt= 6 mod=SqlDtlQueue [MEMORY] hold= 2,756,608 used= 2,720,005 count= 3 avg_used= 906,668 block_cnt= 3 chunk_cnt= 3 mod=SqlDtl1stBuf [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=TransTimeWheel [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=XATimeWheel [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=MediumTabletMap [MEMORY] hold= 2,375,680 used= 2,359,536 count= 2 avg_used= 1,179,768 block_cnt= 2 chunk_cnt= 2 mod=HashBuckLCSta [MEMORY] hold= 2,293,760 used= 2,277,440 count= 10 avg_used= 227,744 block_cnt= 10 chunk_cnt= 5 mod=SQLSessionInfo [MEMORY] hold= 2,248,704 used= 2,228,224 count= 1 avg_used= 2,228,224 block_cnt= 1 chunk_cnt= 1 mod=LogIOCb [MEMORY] hold= 1,777,664 used= 1,770,696 count= 1 avg_used= 1,770,696 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 1,670,592 used= 1,663,936 count= 7 avg_used= 237,705 block_cnt= 7 chunk_cnt= 2 mod=PoolFreeList [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=T3MBucket [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=RewriteRuleMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPlanCac [MEMORY] hold= 958,464 used= 950,272 count= 1 avg_used= 950,272 block_cnt= 1 chunk_cnt= 1 mod=IOWorkerLQ [MEMORY] hold= 933,888 used= 931,072 count= 1 avg_used= 931,072 block_cnt= 1 chunk_cnt= 1 mod=ArcSenderQueue [MEMORY] hold= 829,632 used= 804,864 count= 258 avg_used= 3,119 block_cnt= 130 chunk_cnt= 3 mod=LSMap [MEMORY] hold= 802,944 used= 786,432 count= 256 avg_used= 3,072 block_cnt= 128 chunk_cnt= 3 mod=TabletToLS [MEMORY] hold= 802,816 used= 800,000 count= 1 avg_used= 800,000 block_cnt= 1 chunk_cnt= 1 mod=SqlFltSpanRec [MEMORY] hold= 802,816 used= 800,000 count= 1 avg_used= 800,000 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMon [MEMORY] hold= 786,560 used= 524,352 count= 33 avg_used= 15,889 block_cnt= 33 chunk_cnt= 3 mod=LogAlloc [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 663,552 used= 655,360 count= 1 avg_used= 655,360 block_cnt= 1 chunk_cnt= 1 mod=FetchLog [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DupTbLease [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=FrzTrigger [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DetectorTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=ElectTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTF [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTR [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MdsT [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=OBJLockGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MultiVersionGC [MEMORY] hold= 598,016 used= 590,232 count= 1 avg_used= 590,232 block_cnt= 1 chunk_cnt= 1 mod=DagNetIdMap [MEMORY] hold= 434,176 used= 416,870 count= 4 avg_used= 104,217 block_cnt= 4 chunk_cnt= 3 mod=SqlMemMgr [MEMORY] hold= 432,000 used= 428,544 count= 54 avg_used= 7,936 block_cnt= 54 chunk_cnt= 22 mod=NotParamInfo [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ReplaySrv [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ApplySrv [MEMORY] hold= 385,024 used= 375,520 count= 2 avg_used= 187,760 block_cnt= 2 chunk_cnt= 1 mod=bf_queue [MEMORY] hold= 348,976 used= 338,016 count= 42 avg_used= 8,048 block_cnt= 42 chunk_cnt= 4 mod=[T]ObPerfEventR [MEMORY] hold= 319,488 used= 307,280 count= 2 avg_used= 153,640 block_cnt= 2 chunk_cnt= 2 mod=HashBuckDmId [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=ColUsagHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=DmlStatHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmChe [MEMORY] hold= 232,000 used= 230,144 count= 29 avg_used= 7,936 block_cnt= 29 chunk_cnt= 14 mod=LSIter [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagNetMap [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagMap [MEMORY] hold= 200,704 used= 196,608 count= 64 avg_used= 3,072 block_cnt= 32 chunk_cnt= 4 mod=TxCtxMgr [MEMORY] hold= 193,536 used= 190,848 count= 42 avg_used= 4,544 block_cnt= 42 chunk_cnt= 6 mod=[T]ObTraceEvent [MEMORY] hold= 147,456 used= 139,264 count= 1 avg_used= 139,264 block_cnt= 1 chunk_cnt= 1 mod=RFLTaskQueue [MEMORY] hold= 114,688 used= 106,496 count= 1 avg_used= 106,496 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlMgr [MEMORY] hold= 110,208 used= 109,312 count= 14 avg_used= 7,808 block_cnt= 14 chunk_cnt= 10 mod=HashNodeLCSta [MEMORY] hold= 98,304 used= 51,264 count= 6 avg_used= 8,544 block_cnt= 6 chunk_cnt= 5 mod=[T]ObDSActionAr [MEMORY] hold= 90,112 used= 82,048 count= 1 avg_used= 82,048 block_cnt= 1 chunk_cnt= 1 mod=MetaMemMgr [MEMORY] hold= 73,728 used= 72,736 count= 1 avg_used= 72,736 block_cnt= 1 chunk_cnt= 1 mod=LogSharedQueueT [MEMORY] hold= 64,640 used= 31,888 count= 446 avg_used= 71 block_cnt= 82 chunk_cnt= 20 mod=Coro [MEMORY] hold= 55,104 used= 54,656 count= 7 avg_used= 7,808 block_cnt= 7 chunk_cnt= 6 mod=HashNodePlanCac [MEMORY] hold= 48,832 used= 44,840 count= 2 avg_used= 22,420 block_cnt= 2 chunk_cnt= 2 mod=DynamicFactor [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=Autoincrement [MEMORY] hold= 32,768 used= 25,600 count= 1 avg_used= 25,600 block_cnt= 1 chunk_cnt= 1 mod=TSQLSessionMgr [MEMORY] hold= 32,768 used= 24,688 count= 2 avg_used= 12,344 block_cnt= 2 chunk_cnt= 2 mod=TLD_ClientTask [MEMORY] hold= 32,768 used= 17,408 count= 2 avg_used= 8,704 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTTP [MEMORY] hold= 30,912 used= 28,224 count= 42 avg_used= 672 block_cnt= 18 chunk_cnt= 6 mod=[T]ObWarningBuf [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 24,224 used= 5,952 count= 248 avg_used= 24 block_cnt= 68 chunk_cnt= 18 mod=[T]MemoryContex [MEMORY] hold= 21,760 used= 20,480 count= 20 avg_used= 1,024 block_cnt= 17 chunk_cnt= 12 mod=TGTimer [MEMORY] hold= 20,176 used= 17,472 count= 42 avg_used= 416 block_cnt= 22 chunk_cnt= 4 mod=OMT_Worker [MEMORY] hold= 16,768 used= 8,448 count= 3 avg_used= 2,816 block_cnt= 2 chunk_cnt= 2 mod=BaseLogWriter [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TblCtxIMgr [MEMORY] hold= 16,384 used= 8,448 count= 1 avg_used= 8,448 block_cnt= 1 chunk_cnt= 1 mod=PalfEnv [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TableCtxMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_AssignedMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=backupTaskSched [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TabletStats [MEMORY] hold= 8,192 used= 8,056 count= 2 avg_used= 4,028 block_cnt= 2 chunk_cnt= 2 mod=StoLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=APPLY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DASIDCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LsElectRefInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=REPLAY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ObDMMDL [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LSLocCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ShareBlksMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LCLSender [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LockWaitMgr [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MdsDebugMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 7,088 used= 6,912 count= 2 avg_used= 3,456 block_cnt= 2 chunk_cnt= 2 mod=CACHE_INST [MEMORY] hold= 6,656 used= 4,416 count= 35 avg_used= 126 block_cnt= 35 chunk_cnt= 17 mod=NotParamIdex [MEMORY] hold= 6,416 used= 6,216 count= 3 avg_used= 2,072 block_cnt= 2 chunk_cnt= 2 mod=DetectManager [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResRuleIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=PxPoolBkt [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResGrpIdMap [MEMORY] hold= 4,160 used= 4,096 count= 1 avg_used= 4,096 block_cnt= 1 chunk_cnt= 1 mod=MacroFile [MEMORY] hold= 3,776 used= 3,712 count= 1 avg_used= 3,712 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlDfc [MEMORY] hold= 3,328 used= 3,200 count= 2 avg_used= 1,600 block_cnt= 1 chunk_cnt= 1 mod=PoolArenaArray [MEMORY] hold= 2,304 used= 1,792 count= 8 avg_used= 224 block_cnt= 1 chunk_cnt= 1 mod=LogIOTask [MEMORY] hold= 1,920 used= 1,856 count= 1 avg_used= 1,856 block_cnt= 1 chunk_cnt= 1 mod=LogIOWS [MEMORY] hold= 1,920 used= 1,800 count= 1 avg_used= 1,800 block_cnt= 1 chunk_cnt= 1 mod=PxResMgr [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TabStatMgrLock [MEMORY] hold= 1,600 used= 1,344 count= 4 avg_used= 336 block_cnt= 4 chunk_cnt= 2 mod=NegParamIdex [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=DUP_LS_SET [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GCMemtableMap [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 1,120 used= 1,056 count= 1 avg_used= 1,056 block_cnt= 1 chunk_cnt= 1 mod=LOG_HASH_MAP [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=CheckPointTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TLD_TIMER [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TabShellTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=DiskUsageTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=FlushTimer [MEMORY] hold= 720 used= 648 count= 1 avg_used= 648 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfo [MEMORY] hold= 624 used= 552 count= 1 avg_used= 552 block_cnt= 1 chunk_cnt= 1 mod=LSFreeze [MEMORY] hold= 480 used= 120 count= 5 avg_used= 24 block_cnt= 1 chunk_cnt= 1 mod=FreezeTask [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=HAScheduler [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=Scheduler [MEMORY] hold= 448 used= 384 count= 1 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=MSTXCTX [MEMORY] hold= 336 used= 144 count= 2 avg_used= 72 block_cnt= 2 chunk_cnt= 1 mod=TntSrvObjPool [MEMORY] hold= 320 used= 8 count= 4 avg_used= 2 block_cnt= 4 chunk_cnt= 4 mod=[T]TSIUseWeak [MEMORY] hold= 192 used= 112 count= 1 avg_used= 112 block_cnt= 1 chunk_cnt= 1 mod=Coordinator [MEMORY] hold= 160 used= 16 count= 2 avg_used= 8 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTHandler [MEMORY] hold= 128 used= 56 count= 1 avg_used= 56 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfoMap [MEMORY] hold= 80 used= 16 count= 1 avg_used= 16 block_cnt= 1 chunk_cnt= 1 mod=logservice [MEMORY] hold= 395,343,200 used= 389,702,966 count= 5,983 avg_used= 65,135 mod=SUMMARY [2024-06-25 17:00:18.760352] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=19] [MEMORY] tenant_id= 1003 ctx_id= PLAN_CACHE_CTX_ID hold= 230,686,720 used= 218,693,232 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 204,144,528 used= 187,919,997 count= 9,180 avg_used= 20,470 block_cnt= 9,180 chunk_cnt= 109 mod=SqlPhyPlan [MEMORY] hold= 9,924,384 used= 9,145,448 count= 905 avg_used= 10,105 block_cnt= 905 chunk_cnt= 109 mod=SqlPlanCache [MEMORY] hold= 2,937,472 used= 2,845,056 count= 1,434 avg_used= 1,984 block_cnt= 361 chunk_cnt= 103 mod=CreateContext [MEMORY] hold= 1,237,120 used= 1,196,088 count= 140 avg_used= 8,543 block_cnt= 140 chunk_cnt= 2 mod=SqlPhyPlObj [MEMORY] hold= 448,000 used= 444,416 count= 56 avg_used= 7,936 block_cnt= 56 chunk_cnt= 18 mod=OB_SQLSTAT_OBJ [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=PlanCache [MEMORY] hold= 218,693,232 used= 201,552,605 count= 11,717 avg_used= 17,201 mod=SUMMARY [2024-06-25 17:00:18.760363] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1003 ctx_id= GLIBC hold= 4,194,304 used= 2,327,872 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 2,279,072 used= 1,819,125 count= 6,525 avg_used= 278 block_cnt= 288 chunk_cnt= 2 mod=PlJit [MEMORY] hold= 48,800 used= 45,810 count= 42 avg_used= 1,090 block_cnt= 9 chunk_cnt= 1 mod=PlCodeGen [MEMORY] hold= 2,327,872 used= 1,864,935 count= 6,567 avg_used= 283 mod=SUMMARY [2024-06-25 17:00:18.760371] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1003 ctx_id= CO_STACK hold= 132,120,576 used= 128,507,904 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 128,507,904 used= 128,268,864 count= 249 avg_used= 515,136 block_cnt= 249 chunk_cnt= 63 mod=CoStack [MEMORY] hold= 128,507,904 used= 128,268,864 count= 249 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.760383] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1003 ctx_id= KVSTORE_CACHE_ID hold= 12,582,912 used= 12,582,912 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 12,582,912 used= 0 count= 6 avg_used= 0 block_cnt= 0 chunk_cnt= 0 mod=KvstorCacheMb [MEMORY] hold= 12,582,912 used= 0 count= 6 avg_used= 0 mod=SUMMARY [2024-06-25 17:00:18.760390] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1003 ctx_id= META_OBJ_CTX_ID hold= 2,097,152 used= 401,408 limit= 16,106,127,360 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 401,408 used= 400,064 count= 2 avg_used= 200,032 block_cnt= 2 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 401,408 used= 400,064 count= 2 avg_used= 200,032 mod=SUMMARY [2024-06-25 17:00:18.760400] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1003][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1003 ctx_id= LOB_CTX_ID hold= 2,097,152 used= 65,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LobAllocator [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 mod=SUMMARY [2024-06-25 17:00:18.760645] INFO [LIB] operator() (ob_malloc_allocator.cpp:533) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant: 1004, limit: 724,775,731,200 hold: 622,220,120,064 rpc_hold: 0 cache_hold: 541,409,148,928 cache_used: 541,409,148,928 cache_item_count: 258,164 [MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 20,837,236,736 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MEMSTORE_CTX_ID hold_bytes= 55,475,961,856 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= TRANS_CTX_MGR_ID hold_bytes= 6,291,456 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PLAN_CACHE_CTX_ID hold_bytes= 1,232,424,960 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= GLIBC hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= CO_STACK hold_bytes= 683,671,552 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= META_OBJ_CTX_ID hold_bytes= 165,675,008 limit= 144,955,146,240 [MEMORY] ctx_id= TX_CALLBACK_CTX_ID hold_bytes= 322,961,408 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= LOB_CTX_ID hold_bytes= 2,097,152 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= PS_CACHE_CTX_ID hold_bytes= 6,291,456 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= TX_DATA_TABLE hold_bytes= 2,053,193,728 limit= 9,223,372,036,854,775,807 [MEMORY] ctx_id= MDS_DATA_ID hold_bytes= 20,971,520 limit= 9,223,372,036,854,775,807 [MEMORY][PM] tid= 99915 used= 0 hold= 2,097,152 pm=0x2b3b32656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99916 used= 0 hold= 4,194,304 pm=0x2b3b326d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99917 used= 0 hold= 4,194,304 pm=0x2b3b32752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99920 used= 0 hold= 2,097,152 pm=0x2b3b328d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99921 used= 0 hold= 4,194,304 pm=0x2b3b32952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99925 used= 0 hold= 4,194,304 pm=0x2b3b32d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99926 used= 0 hold= 4,194,304 pm=0x2b3b32dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99927 used= 0 hold= 4,194,304 pm=0x2b3b32e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99928 used= 0 hold= 4,194,304 pm=0x2b3b32ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99929 used= 0 hold= 4,194,304 pm=0x2b3b32f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99930 used= 0 hold= 4,194,304 pm=0x2b3b32fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99931 used= 0 hold= 4,194,304 pm=0x2b3b33056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99932 used= 0 hold= 4,194,304 pm=0x2b3b330d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99933 used= 0 hold= 4,194,304 pm=0x2b3b33152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99934 used= 0 hold= 4,194,304 pm=0x2b3b331d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99935 used= 0 hold= 4,194,304 pm=0x2b3b33256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99936 used= 0 hold= 4,194,304 pm=0x2b3b332d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99937 used= 0 hold= 4,194,304 pm=0x2b3b33352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99938 used= 0 hold= 4,194,304 pm=0x2b3b333d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99939 used= 0 hold= 4,194,304 pm=0x2b3b33456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99940 used= 0 hold= 4,194,304 pm=0x2b3b334d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99941 used= 0 hold= 4,194,304 pm=0x2b3b33552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99942 used= 0 hold= 4,194,304 pm=0x2b3b335d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99943 used= 0 hold= 4,194,304 pm=0x2b3b33656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99944 used= 0 hold= 4,194,304 pm=0x2b3b336d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99945 used= 0 hold= 4,194,304 pm=0x2b3b33752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99946 used= 0 hold= 4,194,304 pm=0x2b3b337d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99947 used= 0 hold= 4,194,304 pm=0x2b3b33856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99948 used= 0 hold= 4,194,304 pm=0x2b3b338d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99949 used= 0 hold= 4,194,304 pm=0x2b3b33952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99950 used= 0 hold= 4,194,304 pm=0x2b3b339d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99951 used= 0 hold= 4,194,304 pm=0x2b3b33a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99952 used= 0 hold= 4,194,304 pm=0x2b3b33ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99953 used= 0 hold= 4,194,304 pm=0x2b3b33b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99954 used= 0 hold= 4,194,304 pm=0x2b3b33bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99955 used= 0 hold= 4,194,304 pm=0x2b3b33e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99956 used= 0 hold= 4,194,304 pm=0x2b3b33ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99957 used= 0 hold= 4,194,304 pm=0x2b3b33f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99958 used= 0 hold= 4,194,304 pm=0x2b3b33fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99959 used= 0 hold= 4,194,304 pm=0x2b3b34456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99960 used= 0 hold= 4,194,304 pm=0x2b3b344d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99961 used= 0 hold= 4,194,304 pm=0x2b3b34552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99962 used= 0 hold= 4,194,304 pm=0x2b3b345d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99963 used= 0 hold= 4,194,304 pm=0x2b3b34656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99964 used= 0 hold= 4,194,304 pm=0x2b3b346d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99965 used= 0 hold= 4,194,304 pm=0x2b3b34752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99966 used= 0 hold= 4,194,304 pm=0x2b3b347d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99967 used= 0 hold= 4,194,304 pm=0x2b3b34856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99968 used= 0 hold= 4,194,304 pm=0x2b3b348d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99969 used= 0 hold= 4,194,304 pm=0x2b3b34952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99970 used= 0 hold= 4,194,304 pm=0x2b3b349d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99971 used= 0 hold= 4,194,304 pm=0x2b3b34a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99972 used= 0 hold= 4,194,304 pm=0x2b3b34ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99973 used= 0 hold= 4,194,304 pm=0x2b3b34b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99974 used= 0 hold= 4,194,304 pm=0x2b3b34bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99975 used= 0 hold= 4,194,304 pm=0x2b3b34c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99976 used= 0 hold= 4,194,304 pm=0x2b3b34cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99977 used= 0 hold= 4,194,304 pm=0x2b3b34d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99978 used= 0 hold= 4,194,304 pm=0x2b3b34dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99979 used= 0 hold= 4,194,304 pm=0x2b3b34e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99980 used= 0 hold= 4,194,304 pm=0x2b3b34ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99981 used= 0 hold= 4,194,304 pm=0x2b3b34f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99982 used= 0 hold= 4,194,304 pm=0x2b3b34fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99983 used= 0 hold= 4,194,304 pm=0x2b3b35056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99984 used= 0 hold= 4,194,304 pm=0x2b3b350d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99985 used= 0 hold= 4,194,304 pm=0x2b3b35152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99986 used= 0 hold= 4,194,304 pm=0x2b3b351d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99987 used= 0 hold= 4,194,304 pm=0x2b3b35256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99988 used= 0 hold= 4,194,304 pm=0x2b3b352d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99989 used= 0 hold= 4,194,304 pm=0x2b3b35352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99990 used= 0 hold= 4,194,304 pm=0x2b3b353d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99991 used= 0 hold= 4,194,304 pm=0x2b3b35456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99992 used= 0 hold= 4,194,304 pm=0x2b3b354d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99993 used= 0 hold= 4,194,304 pm=0x2b3b35552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99994 used= 0 hold= 4,194,304 pm=0x2b3b355d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99995 used= 0 hold= 4,194,304 pm=0x2b3b35656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99996 used= 0 hold= 4,194,304 pm=0x2b3b356d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99997 used= 0 hold= 4,194,304 pm=0x2b3b35752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99998 used= 0 hold= 4,194,304 pm=0x2b3b357d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 99999 used= 0 hold= 4,194,304 pm=0x2b3b35856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100000 used= 0 hold= 4,194,304 pm=0x2b3b358d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100001 used= 0 hold= 4,194,304 pm=0x2b3b35952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100002 used= 0 hold= 4,194,304 pm=0x2b3b359d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100003 used= 0 hold= 4,194,304 pm=0x2b3b35a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100004 used= 0 hold= 4,194,304 pm=0x2b3b35ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100005 used= 0 hold= 4,194,304 pm=0x2b3b35b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100006 used= 0 hold= 4,194,304 pm=0x2b3b35bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100007 used= 0 hold= 4,194,304 pm=0x2b3b35c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100008 used= 0 hold= 4,194,304 pm=0x2b3b35cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100009 used= 0 hold= 4,194,304 pm=0x2b3b35d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100010 used= 0 hold= 4,194,304 pm=0x2b3b35dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100011 used= 0 hold= 4,194,304 pm=0x2b3b35e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100012 used= 0 hold= 4,194,304 pm=0x2b3b35ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100013 used= 0 hold= 4,194,304 pm=0x2b3b35f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100014 used= 0 hold= 4,194,304 pm=0x2b3b35fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100015 used= 0 hold= 4,194,304 pm=0x2b3b36056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100016 used= 0 hold= 4,194,304 pm=0x2b3b360d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100017 used= 0 hold= 4,194,304 pm=0x2b3b36152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100018 used= 0 hold= 4,194,304 pm=0x2b3b361d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100019 used= 0 hold= 4,194,304 pm=0x2b3b36256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100020 used= 0 hold= 4,194,304 pm=0x2b3b362d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100021 used= 0 hold= 4,194,304 pm=0x2b3b36352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100022 used= 0 hold= 4,194,304 pm=0x2b3b363d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100023 used= 0 hold= 4,194,304 pm=0x2b3b36456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100024 used= 0 hold= 4,194,304 pm=0x2b3b364d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100025 used= 0 hold= 4,194,304 pm=0x2b3b36552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100026 used= 0 hold= 4,194,304 pm=0x2b3b365d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100027 used= 0 hold= 4,194,304 pm=0x2b3b36656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100028 used= 0 hold= 4,194,304 pm=0x2b3b366d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100029 used= 0 hold= 4,194,304 pm=0x2b3b36752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100030 used= 0 hold= 4,194,304 pm=0x2b3b367d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100031 used= 0 hold= 4,194,304 pm=0x2b3b36856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100032 used= 0 hold= 4,194,304 pm=0x2b3b368d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100033 used= 0 hold= 4,194,304 pm=0x2b3b36952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100034 used= 0 hold= 4,194,304 pm=0x2b3b369d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100035 used= 0 hold= 4,194,304 pm=0x2b3b36a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100036 used= 0 hold= 4,194,304 pm=0x2b3b36ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100037 used= 0 hold= 4,194,304 pm=0x2b3b36b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100038 used= 0 hold= 4,194,304 pm=0x2b3b36bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100039 used= 0 hold= 4,194,304 pm=0x2b3b36e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100040 used= 0 hold= 4,194,304 pm=0x2b3b36ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100041 used= 0 hold= 4,194,304 pm=0x2b3b36f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100042 used= 0 hold= 4,194,304 pm=0x2b3b36fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100043 used= 0 hold= 4,194,304 pm=0x2b3b37056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100044 used= 0 hold= 4,194,304 pm=0x2b3b370d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100045 used= 0 hold= 4,194,304 pm=0x2b3b37152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100046 used= 0 hold= 4,194,304 pm=0x2b3b371d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100047 used= 0 hold= 4,194,304 pm=0x2b3b37256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100048 used= 0 hold= 4,194,304 pm=0x2b3b372d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100049 used= 0 hold= 4,194,304 pm=0x2b3b37352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100050 used= 0 hold= 4,194,304 pm=0x2b3b373d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100051 used= 0 hold= 4,194,304 pm=0x2b3b37856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100052 used= 0 hold= 4,194,304 pm=0x2b3b378d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100053 used= 0 hold= 4,194,304 pm=0x2b3b37952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100054 used= 0 hold= 4,194,304 pm=0x2b3b379d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100055 used= 0 hold= 4,194,304 pm=0x2b3b37a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100056 used= 0 hold= 4,194,304 pm=0x2b3b37ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100057 used= 0 hold= 4,194,304 pm=0x2b3b37b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100058 used= 0 hold= 4,194,304 pm=0x2b3b37bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100059 used= 0 hold= 4,194,304 pm=0x2b3b37c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100060 used= 0 hold= 4,194,304 pm=0x2b3b37cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100061 used= 0 hold= 4,194,304 pm=0x2b3b37d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100062 used= 0 hold= 4,194,304 pm=0x2b3b37dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100063 used= 0 hold= 4,194,304 pm=0x2b3b37e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100064 used= 0 hold= 4,194,304 pm=0x2b3b37ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100065 used= 0 hold= 4,194,304 pm=0x2b3b37f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100066 used= 0 hold= 4,194,304 pm=0x2b3b37fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100067 used= 0 hold= 4,194,304 pm=0x2b3b38056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100068 used= 0 hold= 4,194,304 pm=0x2b3b380d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100069 used= 0 hold= 4,194,304 pm=0x2b3b38152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100070 used= 0 hold= 4,194,304 pm=0x2b3b381d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100071 used= 0 hold= 4,194,304 pm=0x2b3b38256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100072 used= 0 hold= 4,194,304 pm=0x2b3b382d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100073 used= 0 hold= 4,194,304 pm=0x2b3b38352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100074 used= 0 hold= 4,194,304 pm=0x2b3b383d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100075 used= 0 hold= 4,194,304 pm=0x2b3b38656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100076 used= 0 hold= 4,194,304 pm=0x2b3b386d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100077 used= 0 hold= 4,194,304 pm=0x2b3b38752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100078 used= 0 hold= 4,194,304 pm=0x2b3b387d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100079 used= 0 hold= 4,194,304 pm=0x2b3b38856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100080 used= 0 hold= 4,194,304 pm=0x2b3b388d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100081 used= 0 hold= 4,194,304 pm=0x2b3b38952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100082 used= 0 hold= 4,194,304 pm=0x2b3b389d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100083 used= 0 hold= 4,194,304 pm=0x2b3b38a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100084 used= 0 hold= 4,194,304 pm=0x2b3b38ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100085 used= 0 hold= 4,194,304 pm=0x2b3b38b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100086 used= 0 hold= 4,194,304 pm=0x2b3b38bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100087 used= 0 hold= 4,194,304 pm=0x2b3b38c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100088 used= 0 hold= 4,194,304 pm=0x2b3b38cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100089 used= 0 hold= 4,194,304 pm=0x2b3b38d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100090 used= 0 hold= 4,194,304 pm=0x2b3b38dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100091 used= 0 hold= 4,194,304 pm=0x2b3b38e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100092 used= 0 hold= 4,194,304 pm=0x2b3b38ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100093 used= 0 hold= 4,194,304 pm=0x2b3b38f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100094 used= 0 hold= 4,194,304 pm=0x2b3b38fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100095 used= 0 hold= 4,194,304 pm=0x2b3b39056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100096 used= 0 hold= 4,194,304 pm=0x2b3b390d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100097 used= 0 hold= 4,194,304 pm=0x2b3b39152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100098 used= 0 hold= 4,194,304 pm=0x2b3b391d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100099 used= 0 hold= 4,194,304 pm=0x2b3b39256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100100 used= 0 hold= 4,194,304 pm=0x2b3b392d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100101 used= 0 hold= 4,194,304 pm=0x2b3b39352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100102 used= 0 hold= 4,194,304 pm=0x2b3b393d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100103 used= 0 hold= 4,194,304 pm=0x2b3b39456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100104 used= 0 hold= 4,194,304 pm=0x2b3b394d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100105 used= 0 hold= 4,194,304 pm=0x2b3b39552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100106 used= 0 hold= 4,194,304 pm=0x2b3b395d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100107 used= 0 hold= 4,194,304 pm=0x2b3b39656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100108 used= 0 hold= 4,194,304 pm=0x2b3b396d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100109 used= 0 hold= 4,194,304 pm=0x2b3b39752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100110 used= 0 hold= 4,194,304 pm=0x2b3b397d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100111 used= 0 hold= 4,194,304 pm=0x2b3b39856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100112 used= 0 hold= 4,194,304 pm=0x2b3b398d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100113 used= 0 hold= 4,194,304 pm=0x2b3b39952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100114 used= 0 hold= 4,194,304 pm=0x2b3b399d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100115 used= 0 hold= 4,194,304 pm=0x2b3b39a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100116 used= 0 hold= 4,194,304 pm=0x2b3b39ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100117 used= 0 hold= 4,194,304 pm=0x2b3b39b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100118 used= 0 hold= 4,194,304 pm=0x2b3b39bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100119 used= 0 hold= 4,194,304 pm=0x2b3b39c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100120 used= 0 hold= 4,194,304 pm=0x2b3b39cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100121 used= 0 hold= 4,194,304 pm=0x2b3b39d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100122 used= 0 hold= 4,194,304 pm=0x2b3b39dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100123 used= 0 hold= 4,194,304 pm=0x2b3b39e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100124 used= 0 hold= 4,194,304 pm=0x2b3b39ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100125 used= 0 hold= 4,194,304 pm=0x2b3b39f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100126 used= 0 hold= 4,194,304 pm=0x2b3b39fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100127 used= 0 hold= 4,194,304 pm=0x2b3b3a056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100128 used= 0 hold= 4,194,304 pm=0x2b3b3a0d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100129 used= 0 hold= 4,194,304 pm=0x2b3b3a152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100130 used= 0 hold= 4,194,304 pm=0x2b3b3a1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100131 used= 0 hold= 4,194,304 pm=0x2b3b3a256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100132 used= 0 hold= 4,194,304 pm=0x2b3b3a2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100133 used= 0 hold= 4,194,304 pm=0x2b3b3a352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100134 used= 0 hold= 4,194,304 pm=0x2b3b3a3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100135 used= 0 hold= 4,194,304 pm=0x2b3b3a456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100136 used= 0 hold= 4,194,304 pm=0x2b3b3a4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100137 used= 0 hold= 4,194,304 pm=0x2b3b3a552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100138 used= 0 hold= 4,194,304 pm=0x2b3b3a5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100139 used= 0 hold= 4,194,304 pm=0x2b3b3a656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100140 used= 0 hold= 4,194,304 pm=0x2b3b3a6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100141 used= 0 hold= 4,194,304 pm=0x2b3b3a752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100142 used= 0 hold= 4,194,304 pm=0x2b3b3a7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100143 used= 0 hold= 4,194,304 pm=0x2b3b3a856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100144 used= 0 hold= 4,194,304 pm=0x2b3b3a8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100145 used= 0 hold= 4,194,304 pm=0x2b3b3a952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100146 used= 0 hold= 4,194,304 pm=0x2b3b3a9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100147 used= 0 hold= 4,194,304 pm=0x2b3b3aa56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100148 used= 0 hold= 4,194,304 pm=0x2b3b3aad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100149 used= 0 hold= 4,194,304 pm=0x2b3b3ab52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100150 used= 0 hold= 4,194,304 pm=0x2b3b3abd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100151 used= 0 hold= 4,194,304 pm=0x2b3b3ac56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100152 used= 0 hold= 4,194,304 pm=0x2b3b3acd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100153 used= 0 hold= 4,194,304 pm=0x2b3b3ad52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100154 used= 0 hold= 4,194,304 pm=0x2b3b3add0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100155 used= 0 hold= 4,194,304 pm=0x2b3b3b056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100156 used= 0 hold= 4,194,304 pm=0x2b3b3b0d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100157 used= 0 hold= 4,194,304 pm=0x2b3b3b152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100158 used= 0 hold= 4,194,304 pm=0x2b3b3b1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100159 used= 0 hold= 4,194,304 pm=0x2b3b3b256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100160 used= 0 hold= 4,194,304 pm=0x2b3b3b2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100161 used= 0 hold= 4,194,304 pm=0x2b3b3b352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100162 used= 0 hold= 4,194,304 pm=0x2b3b3b3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100163 used= 0 hold= 4,194,304 pm=0x2b3b3b656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100164 used= 0 hold= 4,194,304 pm=0x2b3b3b6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100165 used= 0 hold= 4,194,304 pm=0x2b3b3b752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100166 used= 0 hold= 4,194,304 pm=0x2b3b3b7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100167 used= 0 hold= 4,194,304 pm=0x2b3b3b856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100168 used= 0 hold= 4,194,304 pm=0x2b3b3b8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100169 used= 0 hold= 4,194,304 pm=0x2b3b3b952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100170 used= 0 hold= 4,194,304 pm=0x2b3b3b9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100171 used= 0 hold= 4,194,304 pm=0x2b3b3ba56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100172 used= 0 hold= 4,194,304 pm=0x2b3b3bad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100173 used= 0 hold= 4,194,304 pm=0x2b3b3bb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100174 used= 0 hold= 4,194,304 pm=0x2b3b3bbd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100175 used= 0 hold= 4,194,304 pm=0x2b3b3bc56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100176 used= 0 hold= 4,194,304 pm=0x2b3b3bcd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100177 used= 0 hold= 4,194,304 pm=0x2b3b3bd52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100178 used= 0 hold= 4,194,304 pm=0x2b3b3bdd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100179 used= 0 hold= 4,194,304 pm=0x2b3b3be56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100180 used= 0 hold= 4,194,304 pm=0x2b3b3bed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100181 used= 0 hold= 4,194,304 pm=0x2b3b3bf52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100182 used= 0 hold= 4,194,304 pm=0x2b3b3bfd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100183 used= 0 hold= 4,194,304 pm=0x2b3b3c056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100184 used= 0 hold= 4,194,304 pm=0x2b3b3c0d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100185 used= 0 hold= 4,194,304 pm=0x2b3b3c152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100186 used= 0 hold= 4,194,304 pm=0x2b3b3c1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100187 used= 0 hold= 4,194,304 pm=0x2b3b3c256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100188 used= 0 hold= 4,194,304 pm=0x2b3b3c2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100189 used= 0 hold= 4,194,304 pm=0x2b3b3c352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100190 used= 0 hold= 4,194,304 pm=0x2b3b3c3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100191 used= 0 hold= 4,194,304 pm=0x2b3b3c456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100192 used= 0 hold= 4,194,304 pm=0x2b3b3c4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100193 used= 0 hold= 4,194,304 pm=0x2b3b3c552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100194 used= 0 hold= 4,194,304 pm=0x2b3b3c5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100195 used= 0 hold= 4,194,304 pm=0x2b3b3c656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100196 used= 0 hold= 4,194,304 pm=0x2b3b3c6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100197 used= 0 hold= 4,194,304 pm=0x2b3b3c752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100198 used= 0 hold= 4,194,304 pm=0x2b3b3c7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100199 used= 0 hold= 4,194,304 pm=0x2b3b3c856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100200 used= 0 hold= 4,194,304 pm=0x2b3b3c8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100201 used= 0 hold= 4,194,304 pm=0x2b3b3c952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100202 used= 0 hold= 4,194,304 pm=0x2b3b3c9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100203 used= 0 hold= 4,194,304 pm=0x2b3b3ca56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100204 used= 0 hold= 4,194,304 pm=0x2b3b3cad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100205 used= 0 hold= 4,194,304 pm=0x2b3b3cb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100206 used= 0 hold= 4,194,304 pm=0x2b3b3cbd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100207 used= 0 hold= 4,194,304 pm=0x2b3b3cc56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100208 used= 0 hold= 4,194,304 pm=0x2b3b3ccd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100209 used= 0 hold= 4,194,304 pm=0x2b3b3cd52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100210 used= 0 hold= 4,194,304 pm=0x2b3b3cdd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100211 used= 0 hold= 4,194,304 pm=0x2b3b3d056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100212 used= 0 hold= 4,194,304 pm=0x2b3b3d0d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100213 used= 0 hold= 4,194,304 pm=0x2b3b3d152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100214 used= 0 hold= 4,194,304 pm=0x2b3b3d1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100215 used= 0 hold= 4,194,304 pm=0x2b3b3d256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100216 used= 0 hold= 4,194,304 pm=0x2b3b3d2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100217 used= 0 hold= 4,194,304 pm=0x2b3b3d352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100218 used= 0 hold= 4,194,304 pm=0x2b3b3d3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100219 used= 0 hold= 4,194,304 pm=0x2b3b3d456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100220 used= 0 hold= 4,194,304 pm=0x2b3b3d4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100221 used= 0 hold= 4,194,304 pm=0x2b3b3d552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100222 used= 0 hold= 4,194,304 pm=0x2b3b3d5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100223 used= 0 hold= 4,194,304 pm=0x2b3b3d656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100224 used= 0 hold= 4,194,304 pm=0x2b3b3d6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100225 used= 0 hold= 4,194,304 pm=0x2b3b3d752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100226 used= 0 hold= 4,194,304 pm=0x2b3b3d7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100227 used= 0 hold= 4,194,304 pm=0x2b3b3da56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100228 used= 0 hold= 4,194,304 pm=0x2b3b3dad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100229 used= 0 hold= 4,194,304 pm=0x2b3b3db52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100230 used= 0 hold= 4,194,304 pm=0x2b3b3dbd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100231 used= 0 hold= 4,194,304 pm=0x2b3b3dc56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100232 used= 0 hold= 4,194,304 pm=0x2b3b3dcd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100233 used= 0 hold= 4,194,304 pm=0x2b3b3dd52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100234 used= 0 hold= 4,194,304 pm=0x2b3b3ddd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100235 used= 0 hold= 4,194,304 pm=0x2b3b3de56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100236 used= 0 hold= 4,194,304 pm=0x2b3b3ded4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100237 used= 0 hold= 4,194,304 pm=0x2b3b3df52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100238 used= 0 hold= 4,194,304 pm=0x2b3b3dfd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100239 used= 0 hold= 4,194,304 pm=0x2b3b3e256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100240 used= 0 hold= 4,194,304 pm=0x2b3b3e2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100241 used= 0 hold= 4,194,304 pm=0x2b3b3e352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100242 used= 0 hold= 4,194,304 pm=0x2b3b3e3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100243 used= 0 hold= 4,194,304 pm=0x2b3b3e456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100244 used= 0 hold= 4,194,304 pm=0x2b3b3e4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100245 used= 0 hold= 4,194,304 pm=0x2b3b3e552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100246 used= 0 hold= 4,194,304 pm=0x2b3b3e5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100247 used= 0 hold= 4,194,304 pm=0x2b3b3e856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100248 used= 0 hold= 4,194,304 pm=0x2b3b3e8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100249 used= 0 hold= 4,194,304 pm=0x2b3b3e952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100250 used= 0 hold= 4,194,304 pm=0x2b3b3e9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100251 used= 0 hold= 4,194,304 pm=0x2b3b3ea56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100252 used= 0 hold= 4,194,304 pm=0x2b3b3ead4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100253 used= 0 hold= 4,194,304 pm=0x2b3b3eb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100254 used= 0 hold= 4,194,304 pm=0x2b3b3ebd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100255 used= 0 hold= 4,194,304 pm=0x2b3b3ec56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100256 used= 0 hold= 4,194,304 pm=0x2b3b3ecd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100257 used= 0 hold= 4,194,304 pm=0x2b3b3ed52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100258 used= 0 hold= 4,194,304 pm=0x2b3b3edd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100259 used= 0 hold= 4,194,304 pm=0x2b3b3ee56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100260 used= 0 hold= 4,194,304 pm=0x2b3b3eed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100261 used= 0 hold= 4,194,304 pm=0x2b3b3ef52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100262 used= 0 hold= 4,194,304 pm=0x2b3b3efd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100263 used= 0 hold= 4,194,304 pm=0x2b3b3f056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100264 used= 0 hold= 4,194,304 pm=0x2b3b3f0d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100265 used= 0 hold= 4,194,304 pm=0x2b3b3f152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100266 used= 0 hold= 4,194,304 pm=0x2b3b3f1d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100267 used= 0 hold= 4,194,304 pm=0x2b3b3f256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100268 used= 0 hold= 4,194,304 pm=0x2b3b3f2d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100269 used= 0 hold= 4,194,304 pm=0x2b3b3f352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100270 used= 0 hold= 4,194,304 pm=0x2b3b3f3d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100271 used= 0 hold= 4,194,304 pm=0x2b3b3f456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100272 used= 0 hold= 4,194,304 pm=0x2b3b3f4d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100273 used= 0 hold= 4,194,304 pm=0x2b3b3f552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100274 used= 0 hold= 4,194,304 pm=0x2b3b3f5d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100275 used= 0 hold= 4,194,304 pm=0x2b3b3f656d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100276 used= 0 hold= 4,194,304 pm=0x2b3b3f6d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100277 used= 0 hold= 4,194,304 pm=0x2b3b3f752d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100278 used= 0 hold= 4,194,304 pm=0x2b3b3f7d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100279 used= 0 hold= 4,194,304 pm=0x2b3b3f856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100280 used= 0 hold= 4,194,304 pm=0x2b3b3f8d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100281 used= 0 hold= 4,194,304 pm=0x2b3b3f952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100282 used= 0 hold= 4,194,304 pm=0x2b3b3f9d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100283 used= 0 hold= 4,194,304 pm=0x2b3b3fa56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100284 used= 0 hold= 4,194,304 pm=0x2b3b3fad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100285 used= 0 hold= 4,194,304 pm=0x2b3b3fb52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100286 used= 0 hold= 4,194,304 pm=0x2b3b3fbd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100287 used= 0 hold= 4,194,304 pm=0x2b3b3fc56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100288 used= 0 hold= 4,194,304 pm=0x2b3b3fcd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100289 used= 0 hold= 4,194,304 pm=0x2b3b3fd52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100290 used= 0 hold= 4,194,304 pm=0x2b3b3fdd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100291 used= 0 hold= 4,194,304 pm=0x2b3b40056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100292 used= 0 hold= 4,194,304 pm=0x2b3b400d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100293 used= 0 hold= 4,194,304 pm=0x2b3b40152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100294 used= 0 hold= 4,194,304 pm=0x2b3b401d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100295 used= 0 hold= 4,194,304 pm=0x2b3b40256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100296 used= 0 hold= 4,194,304 pm=0x2b3b402d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100297 used= 0 hold= 4,194,304 pm=0x2b3b40352d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100298 used= 0 hold= 4,194,304 pm=0x2b3b403d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100299 used= 0 hold= 4,194,304 pm=0x2b3b40456d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100300 used= 0 hold= 4,194,304 pm=0x2b3b404d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100301 used= 0 hold= 4,194,304 pm=0x2b3b40552d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100302 used= 0 hold= 4,194,304 pm=0x2b3b405d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100303 used= 0 hold= 4,194,304 pm=0x2b3b40856d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100304 used= 0 hold= 4,194,304 pm=0x2b3b408d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100305 used= 0 hold= 4,194,304 pm=0x2b3b40952d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100306 used= 0 hold= 4,194,304 pm=0x2b3b409d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100307 used= 0 hold= 4,194,304 pm=0x2b3b40a56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100308 used= 0 hold= 4,194,304 pm=0x2b3b40ad4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100309 used= 0 hold= 4,194,304 pm=0x2b3b40b52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100310 used= 0 hold= 4,194,304 pm=0x2b3b40bd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100311 used= 0 hold= 4,194,304 pm=0x2b3b40c56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100312 used= 0 hold= 4,194,304 pm=0x2b3b40cd4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100313 used= 0 hold= 4,194,304 pm=0x2b3b40d52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100314 used= 0 hold= 4,194,304 pm=0x2b3b40dd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100315 used= 0 hold= 4,194,304 pm=0x2b3b40e56d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100316 used= 0 hold= 4,194,304 pm=0x2b3b40ed4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100317 used= 0 hold= 4,194,304 pm=0x2b3b40f52d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100318 used= 0 hold= 4,194,304 pm=0x2b3b40fd0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100319 used= 0 hold= 4,194,304 pm=0x2b3b41056d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100320 used= 0 hold= 4,194,304 pm=0x2b3b410d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100321 used= 0 hold= 4,194,304 pm=0x2b3b41152d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100322 used= 0 hold= 4,194,304 pm=0x2b3b411d0d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100323 used= 0 hold= 4,194,304 pm=0x2b3b41256d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= 100324 used= 0 hold= 4,194,304 pm=0x2b3b412d4d80 ctx_name=DEFAULT_CTX_ID [MEMORY][PM] tid= summary used= 0 hold= 1,694,498,816 [2024-06-25 17:00:18.760839] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3944143, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=202, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:18.760834] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=39] [MEMORY] tenant_id= 1004 ctx_id= DEFAULT_CTX_ID hold= 20,837,236,736 used= 18,442,559,472 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 15,236,349,952 used= 15,228,866,240 count= 7,286 avg_used= 2,090,154 block_cnt= 7,286 chunk_cnt= 7,286 mod=MysqlRequesReco [MEMORY] hold= 2,014,183,424 used= 2,013,192,192 count= 968 avg_used= 2,079,744 block_cnt= 968 chunk_cnt= 968 mod=CACHE_MAP_NODE [MEMORY] hold= 209,817,600 used= 209,715,200 count= 5 avg_used= 41,943,040 block_cnt= 5 chunk_cnt= 5 mod=LogGroupBuffer [MEMORY] hold= 186,826,752 used= 166,068,224 count= 2,534 avg_used= 65,536 block_cnt= 2,534 chunk_cnt= 426 mod=MinorMergeMgr [MEMORY] hold= 95,846,400 used= 85,581,600 count= 1,300 avg_used= 65,832 block_cnt= 1,300 chunk_cnt= 68 mod=[T]ObSessionDIB [MEMORY] hold= 83,140,608 used= 83,099,648 count= 2 avg_used= 41,549,824 block_cnt= 2 chunk_cnt= 2 mod=ServerObjecPool [MEMORY] hold= 69,648,384 used= 59,359,232 count= 1,256 avg_used= 47,260 block_cnt= 1,256 chunk_cnt= 242 mod=[T]char [MEMORY] hold= 47,259,648 used= 42,008,576 count= 641 avg_used= 65,536 block_cnt= 641 chunk_cnt= 294 mod=MajorMergeMgr [MEMORY] hold= 39,519,904 used= 39,194,896 count= 5,032 avg_used= 7,789 block_cnt= 5,032 chunk_cnt= 399 mod=TLDecoderAlloc [MEMORY] hold= 39,350,272 used= 39,321,760 count= 2 avg_used= 19,660,880 block_cnt= 2 chunk_cnt= 2 mod=HashBuckDmId [MEMORY] hold= 38,772,736 used= 38,401,280 count= 19 avg_used= 2,021,120 block_cnt= 19 chunk_cnt= 19 mod=SqlPlanMon [MEMORY] hold= 37,769,216 used= 37,748,856 count= 1 avg_used= 37,748,856 block_cnt= 1 chunk_cnt= 1 mod=HashBuckDmChe [MEMORY] hold= 27,650,992 used= 27,222,528 count= 5,906 avg_used= 4,609 block_cnt= 5,864 chunk_cnt= 436 mod=WorkerMap [MEMORY] hold= 21,544,480 used= 11,700,168 count= 135,669 avg_used= 86 block_cnt= 331 chunk_cnt= 190 mod=TLDecoderCtx [MEMORY] hold= 19,169,280 used= 17,151,420 count= 260 avg_used= 65,967 block_cnt= 260 chunk_cnt= 65 mod=SqlDtlBuf [MEMORY] hold= 18,616,320 used= 18,493,440 count= 10 avg_used= 1,849,344 block_cnt= 10 chunk_cnt= 9 mod=LogDIOAligned [MEMORY] hold= 18,046,336 used= 17,863,936 count= 2,027 avg_used= 8,812 block_cnt= 2,027 chunk_cnt= 352 mod=ParamCharsType [MEMORY] hold= 16,924,288 used= 16,783,936 count= 10 avg_used= 1,678,393 block_cnt= 10 chunk_cnt= 10 mod=CsSstableReader [MEMORY] hold= 15,524,320 used= 15,480,328 count= 367 avg_used= 42,180 block_cnt= 367 chunk_cnt= 124 mod=TabletMap [MEMORY] hold= 13,064,768 used= 12,959,488 count= 1,633 avg_used= 7,936 block_cnt= 1,633 chunk_cnt= 311 mod=QueryAllocator [MEMORY] hold= 12,816,384 used= 12,763,552 count= 42 avg_used= 303,894 block_cnt= 42 chunk_cnt= 38 mod=IoControl [MEMORY] hold= 11,423,744 used= 11,358,208 count= 4 avg_used= 2,839,552 block_cnt= 4 chunk_cnt= 4 mod=DiskIteratorSto [MEMORY] hold= 8,895,632 used= 8,810,249 count= 79 avg_used= 111,522 block_cnt= 25 chunk_cnt= 5 mod=OmtTenant [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=RCSrv [MEMORY] hold= 8,540,160 used= 8,519,680 count= 1 avg_used= 8,519,680 block_cnt= 1 chunk_cnt= 1 mod=ArcFetchQueue [MEMORY] hold= 8,486,912 used= 8,426,528 count= 37 avg_used= 227,744 block_cnt= 37 chunk_cnt= 36 mod=SQLSessionInfo [MEMORY] hold= 8,257,536 used= 8,240,064 count= 2 avg_used= 4,120,032 block_cnt= 2 chunk_cnt= 2 mod=FreeTbltStream [MEMORY] hold= 8,151,040 used= 8,131,072 count= 1 avg_used= 8,131,072 block_cnt= 1 chunk_cnt= 1 mod=TransService [MEMORY] hold= 7,028,736 used= 6,998,656 count= 107 avg_used= 65,408 block_cnt= 107 chunk_cnt= 29 mod=MemTblObj [MEMORY] hold= 5,980,160 used= 3,118,560 count= 365 avg_used= 8,544 block_cnt= 365 chunk_cnt= 212 mod=[T]ObDSActionAr [MEMORY] hold= 5,644,288 used= 5,625,088 count= 86 avg_used= 65,408 block_cnt= 86 chunk_cnt= 55 mod=DDLKvMgrObj [MEMORY] hold= 4,739,072 used= 4,718,712 count= 1 avg_used= 4,718,712 block_cnt= 1 chunk_cnt= 1 mod=SqlPlanMonMap [MEMORY] hold= 4,137,600 used= 4,080,336 count= 507 avg_used= 8,048 block_cnt= 507 chunk_cnt= 38 mod=[T]ObPerfEventR [MEMORY] hold= 4,096,000 used= 4,063,232 count= 4 avg_used= 1,015,808 block_cnt= 4 chunk_cnt= 4 mod=CommitVersions [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 11 mod=CkptDgnMemCU [MEMORY] hold= 3,763,584 used= 3,705,600 count= 800 avg_used= 4,632 block_cnt= 800 chunk_cnt= 12 mod=CkptDgnMem [MEMORY] hold= 3,145,728 used= 2,228,224 count= 128 avg_used= 17,408 block_cnt= 128 chunk_cnt= 5 mod=SqlDtlQueue [MEMORY] hold= 2,883,584 used= 2,879,744 count= 2 avg_used= 1,439,872 block_cnt= 2 chunk_cnt= 2 mod=SqlFltSpanRec [MEMORY] hold= 2,801,920 used= 2,762,752 count= 608 avg_used= 4,544 block_cnt= 608 chunk_cnt= 82 mod=[T]ObTraceEvent [MEMORY] hold= 2,756,608 used= 2,720,005 count= 3 avg_used= 906,668 block_cnt= 3 chunk_cnt= 2 mod=SqlDtl1stBuf [MEMORY] hold= 2,621,440 used= 2,616,320 count= 40 avg_used= 65,408 block_cnt= 40 chunk_cnt= 33 mod=MemTblMgrObj [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=TransTimeWheel [MEMORY] hold= 2,588,672 used= 2,560,768 count= 4 avg_used= 640,192 block_cnt= 4 chunk_cnt= 2 mod=XATimeWheel [MEMORY] hold= 2,547,712 used= 2,529,856 count= 1 avg_used= 2,529,856 block_cnt= 1 chunk_cnt= 1 mod=RpcStatInfo [MEMORY] hold= 2,490,368 used= 2,468,576 count= 34 avg_used= 72,605 block_cnt= 34 chunk_cnt= 33 mod=bf_queue [MEMORY] hold= 2,379,776 used= 2,359,608 count= 1 avg_used= 2,359,608 block_cnt= 1 chunk_cnt= 1 mod=MediumTabletMap [MEMORY] hold= 2,375,680 used= 2,359,536 count= 2 avg_used= 1,179,768 block_cnt= 2 chunk_cnt= 2 mod=HashBuckLCSta [MEMORY] hold= 2,248,704 used= 2,228,224 count= 1 avg_used= 2,228,224 block_cnt= 1 chunk_cnt= 1 mod=LogIOCb [MEMORY] hold= 2,097,152 used= 2,093,056 count= 32 avg_used= 65,408 block_cnt= 32 chunk_cnt= 32 mod=StorMetaCacheIO [MEMORY] hold= 2,080,768 used= 2,079,744 count= 1 avg_used= 2,079,744 block_cnt= 1 chunk_cnt= 1 mod=LogReplayTask [MEMORY] hold= 2,064,384 used= 2,062,336 count= 2 avg_used= 1,031,168 block_cnt= 2 chunk_cnt= 2 mod=Compressor [MEMORY] hold= 1,925,120 used= 1,885,600 count= 5 avg_used= 377,120 block_cnt= 5 chunk_cnt= 5 mod=ClogGe [MEMORY] hold= 1,777,664 used= 1,770,696 count= 1 avg_used= 1,770,696 block_cnt= 1 chunk_cnt= 1 mod=TableProc [MEMORY] hold= 1,670,592 used= 1,663,936 count= 7 avg_used= 237,705 block_cnt= 7 chunk_cnt= 2 mod=PoolFreeList [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=T3MBucket [MEMORY] hold= 1,581,056 used= 1,572,872 count= 1 avg_used= 1,572,872 block_cnt= 1 chunk_cnt= 1 mod=ResourMapLock [MEMORY] hold= 1,376,256 used= 1,343,488 count= 4 avg_used= 335,872 block_cnt= 4 chunk_cnt= 4 mod=IOWorkerLQ [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPsCache [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPlanCac [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=RewriteRuleMap [MEMORY] hold= 1,187,840 used= 1,179,768 count= 1 avg_used= 1,179,768 block_cnt= 1 chunk_cnt= 1 mod=HashBuckPsInfo [MEMORY] hold= 933,888 used= 931,072 count= 1 avg_used= 931,072 block_cnt= 1 chunk_cnt= 1 mod=ArcSenderQueue [MEMORY] hold= 917,504 used= 915,712 count= 14 avg_used= 65,408 block_cnt= 14 chunk_cnt= 14 mod=TmpBlockManager [MEMORY] hold= 829,632 used= 804,864 count= 258 avg_used= 3,119 block_cnt= 130 chunk_cnt= 2 mod=LSMap [MEMORY] hold= 802,944 used= 786,432 count= 256 avg_used= 3,072 block_cnt= 128 chunk_cnt= 3 mod=TabletToLS [MEMORY] hold= 786,752 used= 524,352 count= 36 avg_used= 14,565 block_cnt= 34 chunk_cnt= 5 mod=LogAlloc [MEMORY] hold= 699,184 used= 693,488 count= 89 avg_used= 7,792 block_cnt= 89 chunk_cnt= 25 mod=CkptDgnMemNode [MEMORY] hold= 692,768 used= 687,104 count= 88 avg_used= 7,808 block_cnt= 88 chunk_cnt= 73 mod=HashNodeLCSta [MEMORY] hold= 663,552 used= 655,360 count= 1 avg_used= 655,360 block_cnt= 1 chunk_cnt= 1 mod=FetchLog [MEMORY] hold= 663,552 used= 659,200 count= 1 avg_used= 659,200 block_cnt= 1 chunk_cnt= 1 mod=MulLevelQueue [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DupTbLease [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTF [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=CoordTR [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=DetectorTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MultiVersionGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=MdsT [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=ElectTimer [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=OBJLockGC [MEMORY] hold= 647,168 used= 640,192 count= 1 avg_used= 640,192 block_cnt= 1 chunk_cnt= 1 mod=FrzTrigger [MEMORY] hold= 614,400 used= 553,040 count= 10 avg_used= 55,304 block_cnt= 10 chunk_cnt= 5 mod=TabletIDSetBkt [MEMORY] hold= 608,960 used= 599,168 count= 151 avg_used= 3,968 block_cnt= 80 chunk_cnt= 30 mod=CkpDgn [MEMORY] hold= 606,208 used= 589,872 count= 2 avg_used= 294,936 block_cnt= 2 chunk_cnt= 1 mod=TmpBlockMap [MEMORY] hold= 598,016 used= 590,232 count= 1 avg_used= 590,232 block_cnt= 1 chunk_cnt= 1 mod=DagNetIdMap [MEMORY] hold= 589,824 used= 569,088 count= 6 avg_used= 94,848 block_cnt= 6 chunk_cnt= 6 mod=ResourceGroup [MEMORY] hold= 511,936 used= 493,990 count= 14 avg_used= 35,285 block_cnt= 14 chunk_cnt= 12 mod=SqlMemMgr [MEMORY] hold= 450,560 used= 422,240 count= 5 avg_used= 84,448 block_cnt= 5 chunk_cnt= 4 mod=LSSvr [MEMORY] hold= 450,560 used= 403,744 count= 11 avg_used= 36,704 block_cnt= 11 chunk_cnt= 11 mod=CommSysVarFac [MEMORY] hold= 447,312 used= 407,904 count= 607 avg_used= 672 block_cnt= 273 chunk_cnt= 64 mod=[T]ObWarningBuf [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ApplySrv [MEMORY] hold= 409,600 used= 401,408 count= 1 avg_used= 401,408 block_cnt= 1 chunk_cnt= 1 mod=ReplaySrv [MEMORY] hold= 370,192 used= 366,976 count= 47 avg_used= 7,808 block_cnt= 47 chunk_cnt= 37 mod=TabletIDSetNode [MEMORY] hold= 314,880 used= 312,320 count= 40 avg_used= 7,808 block_cnt= 40 chunk_cnt= 36 mod=HashNodePlanCac [MEMORY] hold= 310,480 used= 166,424 count= 1,980 avg_used= 84 block_cnt= 162 chunk_cnt= 48 mod=Coro [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=ColUsagHashMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=TmpMap [MEMORY] hold= 303,104 used= 294,936 count= 1 avg_used= 294,936 block_cnt= 1 chunk_cnt= 1 mod=DmlStatHashMap [MEMORY] hold= 291,344 used= 285,064 count= 36 avg_used= 7,918 block_cnt= 36 chunk_cnt= 14 mod=TabletStats [MEMORY] hold= 266,112 used= 261,888 count= 66 avg_used= 3,968 block_cnt= 45 chunk_cnt= 30 mod=TinyAllocator [MEMORY] hold= 264,000 used= 261,888 count= 33 avg_used= 7,936 block_cnt= 33 chunk_cnt= 30 mod=SqlSession [MEMORY] hold= 262,144 used= 261,632 count= 4 avg_used= 65,408 block_cnt= 4 chunk_cnt= 4 mod=TmpPageCache [MEMORY] hold= 262,144 used= 261,632 count= 4 avg_used= 65,408 block_cnt= 4 chunk_cnt= 4 mod=LockMemObj [MEMORY] hold= 256,016 used= 253,952 count= 32 avg_used= 7,936 block_cnt= 32 chunk_cnt= 32 mod=LSIter [MEMORY] hold= 249,408 used= 211,752 count= 519 avg_used= 408 block_cnt= 516 chunk_cnt= 90 mod=[T]ObTLDecoderC [MEMORY] hold= 246,160 used= 212,992 count= 512 avg_used= 416 block_cnt= 68 chunk_cnt= 13 mod=OMT_Worker [MEMORY] hold= 245,760 used= 243,840 count= 5 avg_used= 48,768 block_cnt= 5 chunk_cnt= 4 mod=palf_env [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagNetMap [MEMORY] hold= 204,800 used= 196,744 count= 1 avg_used= 196,744 block_cnt= 1 chunk_cnt= 1 mod=DagMap [MEMORY] hold= 200,832 used= 196,608 count= 64 avg_used= 3,072 block_cnt= 32 chunk_cnt= 4 mod=TxCtxMgr [MEMORY] hold= 147,456 used= 139,264 count= 1 avg_used= 139,264 block_cnt= 1 chunk_cnt= 1 mod=RFLTaskQueue [MEMORY] hold= 147,456 used= 131,072 count= 2 avg_used= 65,536 block_cnt= 2 chunk_cnt= 2 mod=SHADiagMgr [MEMORY] hold= 147,456 used= 131,072 count= 2 avg_used= 65,536 block_cnt= 2 chunk_cnt= 2 mod=SuspectInfoMgr [MEMORY] hold= 128,880 used= 97,740 count= 11 avg_used= 8,885 block_cnt= 11 chunk_cnt= 11 mod=SqlSessiQuerSql [MEMORY] hold= 126,048 used= 31,200 count= 1,300 avg_used= 24 block_cnt= 192 chunk_cnt= 51 mod=[T]MemoryContex [MEMORY] hold= 114,688 used= 106,496 count= 1 avg_used= 106,496 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlMgr [MEMORY] hold= 109,088 used= 108,192 count= 14 avg_used= 7,728 block_cnt= 14 chunk_cnt= 12 mod=HashNodePsCache [MEMORY] hold= 98,304 used= 94,976 count= 1 avg_used= 94,976 block_cnt= 1 chunk_cnt= 1 mod=PxPool [MEMORY] hold= 90,112 used= 82,048 count= 1 avg_used= 82,048 block_cnt= 1 chunk_cnt= 1 mod=MetaMemMgr [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 11 mod=PlanVaIdx [MEMORY] hold= 88,000 used= 87,296 count= 11 avg_used= 7,936 block_cnt= 11 chunk_cnt= 11 mod=SqlSessiVarMap [MEMORY] hold= 81,920 used= 57,920 count= 5 avg_used= 11,584 block_cnt= 5 chunk_cnt= 4 mod=LogApplyStatus [MEMORY] hold= 81,920 used= 68,480 count= 5 avg_used= 13,696 block_cnt= 5 chunk_cnt= 4 mod=LogReplayStatus [MEMORY] hold= 81,920 used= 51,240 count= 5 avg_used= 10,248 block_cnt= 5 chunk_cnt= 5 mod=TabletSvrBucket [MEMORY] hold= 80,000 used= 79,360 count= 10 avg_used= 7,936 block_cnt= 10 chunk_cnt= 8 mod=LSLocCache [MEMORY] hold= 78,720 used= 78,080 count= 10 avg_used= 7,808 block_cnt= 10 chunk_cnt= 9 mod=HashNodePsInfo [MEMORY] hold= 73,728 used= 65,536 count= 1 avg_used= 65,536 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisMgr [MEMORY] hold= 73,728 used= 66,176 count= 1 avg_used= 66,176 block_cnt= 1 chunk_cnt= 1 mod=Rpc [MEMORY] hold= 73,728 used= 72,736 count= 1 avg_used= 72,736 block_cnt= 1 chunk_cnt= 1 mod=LogSharedQueueT [MEMORY] hold= 73,536 used= 73,344 count= 2 avg_used= 36,672 block_cnt= 2 chunk_cnt= 2 mod=FlushMeta [MEMORY] hold= 65,984 used= 56,896 count= 140 avg_used= 406 block_cnt= 130 chunk_cnt= 47 mod=NegParamIdex [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxDataMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=TxCtxMemObj [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=FlushLog [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=DDLKVObj [MEMORY] hold= 64,000 used= 63,488 count= 8 avg_used= 7,936 block_cnt= 8 chunk_cnt= 5 mod=LCLSender [MEMORY] hold= 63,744 used= 63,232 count= 8 avg_used= 7,904 block_cnt= 8 chunk_cnt= 8 mod=SqlSessionSbloc [MEMORY] hold= 62,848 used= 62,336 count= 8 avg_used= 7,792 block_cnt= 8 chunk_cnt= 7 mod=CkptDgnCUNode [MEMORY] hold= 56,000 used= 55,552 count= 7 avg_used= 7,936 block_cnt= 7 chunk_cnt= 6 mod=NotParamInfo [MEMORY] hold= 48,832 used= 44,840 count= 2 avg_used= 22,420 block_cnt= 2 chunk_cnt= 2 mod=DynamicFactor [MEMORY] hold= 42,240 used= 41,472 count= 12 avg_used= 3,456 block_cnt= 12 chunk_cnt= 7 mod=CACHE_INST [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=SuspectInfoBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=DagWarnHisBkt [MEMORY] hold= 40,960 used= 37,032 count= 1 avg_used= 37,032 block_cnt= 1 chunk_cnt= 1 mod=Autoincrement [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 4 mod=MatchLsnMap [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 5 mod=MdsTableMgr [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 2 mod=ObDMMDL [MEMORY] hold= 32,768 used= 24,688 count= 2 avg_used= 12,344 block_cnt= 2 chunk_cnt= 1 mod=TLD_ClientTask [MEMORY] hold= 32,768 used= 17,408 count= 2 avg_used= 8,704 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTTP [MEMORY] hold= 32,768 used= 25,600 count= 1 avg_used= 25,600 block_cnt= 1 chunk_cnt= 1 mod=TSQLSessionMgr [MEMORY] hold= 32,000 used= 31,744 count= 4 avg_used= 7,936 block_cnt= 4 chunk_cnt= 4 mod=ShareBlksMap [MEMORY] hold= 27,200 used= 25,600 count= 25 avg_used= 1,024 block_cnt= 17 chunk_cnt= 13 mod=TGTimer [MEMORY] hold= 24,864 used= 572 count= 286 avg_used= 2 block_cnt= 224 chunk_cnt= 158 mod=[T]TSIUseWeak [MEMORY] hold= 24,576 used= 16,384 count= 1 avg_used= 16,384 block_cnt= 1 chunk_cnt= 1 mod=SlogWriteBuffer [MEMORY] hold= 23,616 used= 23,424 count= 3 avg_used= 7,808 block_cnt= 3 chunk_cnt= 2 mod=DirToBlk [MEMORY] hold= 23,520 used= 23,160 count= 5 avg_used= 4,632 block_cnt= 5 chunk_cnt= 4 mod=TTLTaskBucket [MEMORY] hold= 16,768 used= 8,448 count= 3 avg_used= 2,816 block_cnt= 2 chunk_cnt= 2 mod=BaseLogWriter [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TblCtxIMgr [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_TableCtxMgr [MEMORY] hold= 16,384 used= 10,248 count= 1 avg_used= 10,248 block_cnt= 1 chunk_cnt= 1 mod=TmpMemBlkMgr [MEMORY] hold= 16,384 used= 8,448 count= 1 avg_used= 8,448 block_cnt= 1 chunk_cnt= 1 mod=PalfEnv [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=TLD_AssignedMgr [MEMORY] hold= 16,384 used= 8,192 count= 1 avg_used= 8,192 block_cnt= 1 chunk_cnt= 1 mod=SlogNopLog [MEMORY] hold= 16,384 used= 12,344 count= 1 avg_used= 12,344 block_cnt= 1 chunk_cnt= 1 mod=backupTaskSched [MEMORY] hold= 15,744 used= 15,616 count= 2 avg_used= 7,808 block_cnt= 2 chunk_cnt= 2 mod=SuspectInfoNode [MEMORY] hold= 15,680 used= 15,360 count= 5 avg_used= 3,072 block_cnt= 5 chunk_cnt= 5 mod=LockMemtableMgr [MEMORY] hold= 12,576 used= 12,440 count= 2 avg_used= 6,220 block_cnt= 2 chunk_cnt= 2 mod=PxPoolBkt [MEMORY] hold= 10,368 used= 10,240 count= 2 avg_used= 5,120 block_cnt= 2 chunk_cnt= 2 mod=PoolArenaArray [MEMORY] hold= 10,240 used= 10,112 count= 2 avg_used= 5,056 block_cnt= 1 chunk_cnt= 1 mod=HAScheduler [MEMORY] hold= 10,240 used= 10,112 count= 2 avg_used= 5,056 block_cnt= 1 chunk_cnt= 1 mod=Scheduler [MEMORY] hold= 9,296 used= 9,160 count= 2 avg_used= 4,580 block_cnt= 2 chunk_cnt= 2 mod=StoLocCache [MEMORY] hold= 8,080 used= 7,720 count= 5 avg_used= 1,544 block_cnt= 5 chunk_cnt= 4 mod=TXBFRetryId [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=MdsDebugMap [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=APPLY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=IORunners [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=REPLAY_STATUS [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LockWaitMgr [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=ls_id [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LsElectRefInfo [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=DASIDCache [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=PurgeThrottle [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=FetchLogTask [MEMORY] hold= 7,856 used= 7,792 count= 1 avg_used= 7,792 block_cnt= 1 chunk_cnt= 1 mod=ColUsagNode [MEMORY] hold= 7,776 used= 7,712 count= 1 avg_used= 7,712 block_cnt= 1 chunk_cnt= 1 mod=DmlStatNode [MEMORY] hold= 7,488 used= 7,424 count= 1 avg_used= 7,424 block_cnt= 1 chunk_cnt= 1 mod=LogIOWS [MEMORY] hold= 6,416 used= 6,216 count= 3 avg_used= 2,072 block_cnt= 2 chunk_cnt= 2 mod=DetectManager [MEMORY] hold= 5,936 used= 4,032 count= 28 avg_used= 144 block_cnt= 25 chunk_cnt= 16 mod=ObTxLSLogCb [MEMORY] hold= 5,632 used= 4,664 count= 11 avg_used= 424 block_cnt= 10 chunk_cnt= 10 mod=SequenceMap [MEMORY] hold= 5,552 used= 4,664 count= 11 avg_used= 424 block_cnt= 11 chunk_cnt= 10 mod=SequenceIdMap [MEMORY] hold= 5,504 used= 4,664 count= 11 avg_used= 424 block_cnt= 10 chunk_cnt= 9 mod=PackStateMap [MEMORY] hold= 5,456 used= 4,664 count= 11 avg_used= 424 block_cnt= 10 chunk_cnt= 10 mod=ContextsMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResGrpIdMap [MEMORY] hold= 4,704 used= 4,632 count= 1 avg_used= 4,632 block_cnt= 1 chunk_cnt= 1 mod=ResRuleIdMap [MEMORY] hold= 4,240 used= 3,880 count= 5 avg_used= 776 block_cnt= 5 chunk_cnt= 5 mod=DepTabletSet [MEMORY] hold= 4,160 used= 4,096 count= 1 avg_used= 4,096 block_cnt= 1 chunk_cnt= 1 mod=MacroFile [MEMORY] hold= 3,776 used= 3,712 count= 1 avg_used= 3,712 block_cnt= 1 chunk_cnt= 1 mod=SqlDtlDfc [MEMORY] hold= 2,720 used= 2,360 count= 5 avg_used= 472 block_cnt= 4 chunk_cnt= 3 mod=Election [MEMORY] hold= 2,688 used= 2,304 count= 6 avg_used= 384 block_cnt= 1 chunk_cnt= 1 mod=TableLockNode [MEMORY] hold= 2,352 used= 1,792 count= 8 avg_used= 224 block_cnt= 2 chunk_cnt= 2 mod=LogIOTask [MEMORY] hold= 2,320 used= 1,664 count= 8 avg_used= 208 block_cnt= 8 chunk_cnt= 8 mod=[T]ObMacroBlock [MEMORY] hold= 1,872 used= 1,800 count= 1 avg_used= 1,800 block_cnt= 1 chunk_cnt= 1 mod=PxResMgr [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=MSTXCTX [MEMORY] hold= 1,616 used= 1,544 count= 1 avg_used= 1,544 block_cnt= 1 chunk_cnt= 1 mod=TabStatMgrLock [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=DUP_LS_SET [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GROUP_INDEX_MAP [MEMORY] hold= 1,344 used= 1,272 count= 1 avg_used= 1,272 block_cnt= 1 chunk_cnt= 1 mod=GCMemtableMap [MEMORY] hold= 1,280 used= 1,088 count= 3 avg_used= 362 block_cnt= 3 chunk_cnt= 3 mod=MustBePosiIdx [MEMORY] hold= 1,152 used= 1,080 count= 1 avg_used= 1,080 block_cnt= 1 chunk_cnt= 1 mod=ModuleInitCtx [MEMORY] hold= 1,120 used= 1,056 count= 1 avg_used= 1,056 block_cnt= 1 chunk_cnt= 1 mod=LOG_HASH_MAP [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=FlushTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TabShellTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=TLD_TIMER [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=DiskUsageTimer [MEMORY] hold= 1,088 used= 1,024 count= 1 avg_used= 1,024 block_cnt= 1 chunk_cnt= 1 mod=CheckPointTimer [MEMORY] hold= 1,056 used= 320 count= 10 avg_used= 32 block_cnt= 9 chunk_cnt= 8 mod=PalfFSCbNode [MEMORY] hold= 944 used= 520 count= 5 avg_used= 104 block_cnt= 5 chunk_cnt= 5 mod=RedoLogBuckLock [MEMORY] hold= 720 used= 648 count= 1 avg_used= 648 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfo [MEMORY] hold= 624 used= 552 count= 1 avg_used= 552 block_cnt= 1 chunk_cnt= 1 mod=LSFreeze [MEMORY] hold= 576 used= 205 count= 5 avg_used= 41 block_cnt= 5 chunk_cnt= 4 mod=KeepAliveBuf [MEMORY] hold= 496 used= 160 count= 5 avg_used= 32 block_cnt= 5 chunk_cnt= 5 mod=PalfRCCbNode [MEMORY] hold= 496 used= 160 count= 5 avg_used= 32 block_cnt= 5 chunk_cnt= 4 mod=RebuildCbNode [MEMORY] hold= 480 used= 120 count= 5 avg_used= 24 block_cnt= 1 chunk_cnt= 1 mod=FreezeTask [MEMORY] hold= 368 used= 186 count= 2 avg_used= 93 block_cnt= 2 chunk_cnt= 2 mod=TxLSLogBuf [MEMORY] hold= 336 used= 144 count= 2 avg_used= 72 block_cnt= 2 chunk_cnt= 1 mod=TntSrvObjPool [MEMORY] hold= 176 used= 112 count= 1 avg_used= 112 block_cnt= 1 chunk_cnt= 1 mod=Coordinator [MEMORY] hold= 160 used= 16 count= 2 avg_used= 8 block_cnt= 2 chunk_cnt= 1 mod=ObLogEXTHandler [MEMORY] hold= 128 used= 56 count= 1 avg_used= 56 block_cnt= 1 chunk_cnt= 1 mod=ResRuleInfoMap [MEMORY] hold= 80 used= 16 count= 1 avg_used= 16 block_cnt= 1 chunk_cnt= 1 mod=logservice [MEMORY] hold= 18,442,559,472 used= 18,367,596,231 count= 175,710 avg_used= 104,533 mod=SUMMARY [2024-06-25 17:00:18.760868] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=29] [MEMORY] tenant_id= 1004 ctx_id= MEMSTORE_CTX_ID hold= 55,475,961,856 used= 55,042,555,904 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 55,042,555,904 used= 55,017,372,648 count= 26,453 avg_used= 2,079,816 block_cnt= 26,453 chunk_cnt= 26,453 mod=Memstore [MEMORY] hold= 55,042,555,904 used= 55,017,372,648 count= 26,453 avg_used= 2,079,816 mod=SUMMARY [2024-06-25 17:00:18.760876] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1004 ctx_id= TRANS_CTX_MGR_ID hold= 6,291,456 used= 3,645,440 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 3,645,440 used= 3,612,480 count= 5 avg_used= 722,496 block_cnt= 5 chunk_cnt= 3 mod=PartTranCtxMgr [MEMORY] hold= 3,645,440 used= 3,612,480 count= 5 avg_used= 722,496 mod=SUMMARY [2024-06-25 17:00:18.760885] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1004 ctx_id= PLAN_CACHE_CTX_ID hold= 1,232,424,960 used= 1,205,544,208 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 1,090,557,456 used= 1,016,422,555 count= 46,115 avg_used= 22,041 block_cnt= 46,115 chunk_cnt= 566 mod=SqlPhyPlan [MEMORY] hold= 66,337,664 used= 61,810,624 count= 3,844 avg_used= 16,079 block_cnt= 3,844 chunk_cnt= 545 mod=SqlPlanCache [MEMORY] hold= 28,889,920 used= 28,656,896 count= 3,611 avg_used= 7,936 block_cnt= 3,611 chunk_cnt= 207 mod=OB_SQLSTAT_OBJ [MEMORY] hold= 18,520,320 used= 17,937,344 count= 9,041 avg_used= 1,984 block_cnt= 2,270 chunk_cnt= 510 mod=CreateContext [MEMORY] hold= 1,237,120 used= 1,196,088 count= 140 avg_used= 8,543 block_cnt= 140 chunk_cnt= 2 mod=SqlPhyPlObj [MEMORY] hold= 1,728 used= 1,600 count= 2 avg_used= 800 block_cnt= 1 chunk_cnt= 1 mod=PlanCache [MEMORY] hold= 1,205,544,208 used= 1,126,025,107 count= 62,753 avg_used= 17,943 mod=SUMMARY [2024-06-25 17:00:18.760896] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1004 ctx_id= GLIBC hold= 4,194,304 used= 2,327,824 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 2,278,992 used= 1,819,125 count= 6,525 avg_used= 278 block_cnt= 290 chunk_cnt= 2 mod=PlJit [MEMORY] hold= 48,832 used= 45,810 count= 42 avg_used= 1,090 block_cnt= 13 chunk_cnt= 1 mod=PlCodeGen [MEMORY] hold= 2,327,824 used= 1,864,935 count= 6,567 avg_used= 283 mod=SUMMARY [2024-06-25 17:00:18.760904] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1004 ctx_id= CO_STACK hold= 683,671,552 used= 670,924,800 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 670,924,800 used= 669,676,800 count= 1,300 avg_used= 515,136 block_cnt= 1,300 chunk_cnt= 326 mod=CoStack [MEMORY] hold= 670,924,800 used= 669,676,800 count= 1,300 avg_used= 515,136 mod=SUMMARY [2024-06-25 17:00:18.760913] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1004 ctx_id= KVSTORE_CACHE_ID hold= 541,409,148,928 used= 541,409,148,928 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 541,409,148,928 used= 0 count= 258,164 avg_used= 0 block_cnt= 0 chunk_cnt= 0 mod=KvstorCacheMb [MEMORY] hold= 541,409,148,928 used= 0 count= 258,164 avg_used= 0 mod=SUMMARY [2024-06-25 17:00:18.760922] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1004 ctx_id= META_OBJ_CTX_ID hold= 165,675,008 used= 162,308,096 limit= 144,955,146,240 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 147,456,000 used= 131,182,000 count= 2,000 avg_used= 65,591 block_cnt= 2,000 chunk_cnt= 74 mod=L_TabletPool [MEMORY] hold= 14,450,688 used= 14,389,760 count= 220 avg_used= 65,408 block_cnt= 220 chunk_cnt= 11 mod=N_TabletPool [MEMORY] hold= 401,408 used= 400,064 count= 2 avg_used= 200,032 block_cnt= 2 chunk_cnt= 1 mod=PoolFreeList [MEMORY] hold= 162,308,096 used= 145,971,824 count= 2,222 avg_used= 65,693 mod=SUMMARY [2024-06-25 17:00:18.760930] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1004 ctx_id= TX_CALLBACK_CTX_ID hold= 322,961,408 used= 293,306,816 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 293,306,816 used= 290,941,696 count= 36,661 avg_used= 7,936 block_cnt= 36,661 chunk_cnt= 154 mod=MemtableCallbac [MEMORY] hold= 293,306,816 used= 290,941,696 count= 36,661 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.760939] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1004 ctx_id= LOB_CTX_ID hold= 2,097,152 used= 73,536 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 65,536 used= 65,408 count= 1 avg_used= 65,408 block_cnt= 1 chunk_cnt= 1 mod=LobAllocator [MEMORY] hold= 8,000 used= 7,936 count= 1 avg_used= 7,936 block_cnt= 1 chunk_cnt= 1 mod=LobPersist [MEMORY] hold= 73,536 used= 73,344 count= 2 avg_used= 36,672 mod=SUMMARY [2024-06-25 17:00:18.760947] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=4] [MEMORY] tenant_id= 1004 ctx_id= PS_CACHE_CTX_ID hold= 6,291,456 used= 3,809,376 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 3,809,376 used= 3,267,545 count= 1,998 avg_used= 1,635 block_cnt= 351 chunk_cnt= 3 mod=SqlPsCache [MEMORY] hold= 3,809,376 used= 3,267,545 count= 1,998 avg_used= 1,635 mod=SUMMARY [2024-06-25 17:00:18.760958] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=3] [MEMORY] tenant_id= 1004 ctx_id= TX_DATA_TABLE hold= 2,053,193,728 used= 1,985,541,824 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 1,953,873,472 used= 1,938,121,984 count= 244,219 avg_used= 7,936 block_cnt= 244,219 chunk_cnt= 967 mod=TX_DATA_SLICE [MEMORY] hold= 31,628,352 used= 31,497,280 count= 15 avg_used= 2,099,818 block_cnt= 15 chunk_cnt= 12 mod=MEMTABLE_ARENA [MEMORY] hold= 40,000 used= 39,680 count= 5 avg_used= 7,936 block_cnt= 5 chunk_cnt= 2 mod=TX_DATA_ARENA [MEMORY] hold= 1,985,541,824 used= 1,969,658,944 count= 244,239 avg_used= 8,064 mod=SUMMARY [2024-06-25 17:00:18.760968] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [62165][MemDumpTimer][T1004][Y0-0000000000000000-0-0] [lt=5] [MEMORY] tenant_id= 1004 ctx_id= MDS_DATA_ID hold= 20,971,520 used= 256,000 limit= 9,223,372,036,854,775,807 [MEMORY] idle_size= 0 free_size= 0 [MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0 [MEMORY] hold= 256,000 used= 253,952 count= 32 avg_used= 7,936 block_cnt= 32 chunk_cnt= 10 mod=MdsTable [MEMORY] hold= 256,000 used= 253,952 count= 32 avg_used= 7,936 mod=SUMMARY [2024-06-25 17:00:18.761008] INFO [STORAGE] print_tenant_usage (ob_tenant_memory_printer.cpp:125) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=4] ====== tenants memory info ====== === TENANTS MEMORY INFO === all_tenants_memstore_used= 55,765,368,832, divisive_memory_used= 462,983,168 [TENANT_MEMORY] tenant_id= 500 mem_tenant_limit= 9,223,372,036,854,775,807 mem_tenant_hold= 3,474,333,696 kv_cache_mem= 0 [TENANT_MEMORY] tenant_id= 508 mem_tenant_limit= 1,073,741,824 mem_tenant_hold= 40,296,448 kv_cache_mem= 0 [TENANT_MEMORY] tenant_id= 1 now= 1,719,306,018,758,995 active_memstore_used= 285,212,672 total_memstore_used= 285,212,672 total_memstore_hold= 285,212,672 memstore_freeze_trigger_limit= 4,039,953,600 memstore_limit= 20,199,768,050 mem_tenant_limit= 40,399,536,128 mem_tenant_hold= 17,737,887,744 kv_cache_mem= 14,854,127,616 max_mem_memstore_can_get_now= 0 memstore_alloc_pos= 19,216,203,776 memstore_frozen_pos= 18,930,991,104 memstore_reclaimed_pos= 18,930,991,104 [TENANT_MEMORY] tenant_id= 1,001 now= 1,719,306,018,758,995 active_memstore_used= 0 total_memstore_used= 0 total_memstore_hold= 0 memstore_freeze_trigger_limit= 107,374,180 memstore_limit= 536,870,900 mem_tenant_limit= 1,073,741,824 mem_tenant_hold= 343,166,976 kv_cache_mem= 14,680,064 max_mem_memstore_can_get_now= 0 memstore_alloc_pos= 0 memstore_frozen_pos= 0 memstore_reclaimed_pos= 0 [TENANT_MEMORY] tenant_id= 1,002 now= 1,719,306,018,758,995 active_memstore_used= 4,194,304 total_memstore_used= 4,194,304 total_memstore_hold= 4,194,304 memstore_freeze_trigger_limit= 53,542,900 memstore_limit= 536,870,900 mem_tenant_limit= 1,073,741,824 mem_tenant_hold= 933,953,536 kv_cache_mem= 123,731,968 max_mem_memstore_can_get_now= 0 memstore_alloc_pos= 308,054,851,584 memstore_frozen_pos= 308,050,657,280 memstore_reclaimed_pos= 308,050,657,280 [TENANT_MEMORY] tenant_id= 1,003 now= 1,719,306,018,760,043 active_memstore_used= 0 total_memstore_used= 0 total_memstore_hold= 0 memstore_freeze_trigger_limit= 8,053,063,680 memstore_limit= 40,265,318,400 mem_tenant_limit= 80,530,636,800 mem_tenant_hold= 845,557,760 kv_cache_mem= 12,582,912 max_mem_memstore_can_get_now= 0 memstore_alloc_pos= 0 memstore_frozen_pos= 0 memstore_reclaimed_pos= 0 [TENANT_MEMORY] tenant_id= 1,004 now= 1,719,306,018,760,043 active_memstore_used= 55,475,961,856 total_memstore_used= 55,475,961,856 total_memstore_hold= 55,475,961,856 memstore_freeze_trigger_limit= 72,477,573,120 memstore_limit= 362,387,865,600 mem_tenant_limit= 724,775,731,200 mem_tenant_hold= 622,220,120,064 kv_cache_mem= 541,409,148,928 max_mem_memstore_can_get_now= 0 memstore_alloc_pos= 58,193,176,690,688 memstore_frozen_pos= 58,137,700,728,832 memstore_reclaimed_pos= 58,137,700,728,832 [2024-06-25 17:00:18.761071] INFO [STORAGE] print_tenant_usage (ob_tenant_memory_printer.cpp:172) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=7] [CHUNK_MGR] free=182 pushes=10235828566 pops=10235828384 limit=939,524,096,000 hold=646,107,275,264 total_hold=646,765,871,104 used=645,595,316,224 freelist_hold= 511,959,040 large_freelist_hold= 130,277,376 maps= 5,113,416 unmaps= 4,806,942 large_maps= 2,211 large_unmaps= 2,011 huge_maps= 4,753,214 huge_unmaps= 4,753,121 memalign=0 resident_size=647,414,452,224 virtual_memory_used=647,946,121,216 [2024-06-25 17:00:18.761131] INFO print (ob_malloc_time_monitor.cpp:57) [62165][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=8] [MALLOC_TIME_MONITOR] show the distribution of ob_malloc's cost_time [MALLOC_TIME_MONITOR] [ 0, 10): delta_total_cost_time= 2621021, delta_count= 33571495, avg_cost_time= 0 [MALLOC_TIME_MONITOR] [ 10, 100): delta_total_cost_time= 945, delta_count= 44, avg_cost_time= 21 [MALLOC_TIME_MONITOR] [ 100, 1000): delta_total_cost_time= 4827, delta_count= 13, avg_cost_time= 371 [MALLOC_TIME_MONITOR] [ 1000, 10000): delta_total_cost_time= 0, delta_count= 0, avg_cost_time= 0 [MALLOC_TIME_MONITOR] [ 10000, 100000): delta_total_cost_time= 0, delta_count= 0, avg_cost_time= 0 [MALLOC_TIME_MONITOR] [ 100000, 1000000): delta_total_cost_time= 0, delta_count= 0, avg_cost_time= 0 [MALLOC_TIME_MONITOR] [ 1000000, 9223372036854775807): delta_total_cost_time= 0, delta_count= 0, avg_cost_time= 0 [2024-06-25 17:00:18.762227] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=4] Dumping All Cache Objs(alloc_obj_list.count()=717, alloc_obj_list=[{obj_id:14, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176366}, {obj_id:16, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:449104}, {obj_id:17, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303715}, {obj_id:18, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:248080}, {obj_id:19, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271604}, {obj_id:20, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216033}, {obj_id:21, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191870}, {obj_id:22, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:23, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179794}, {obj_id:24, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230228}, {obj_id:25, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188271}, {obj_id:26, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196930}, {obj_id:28, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174576}, {obj_id:29, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301933}, {obj_id:33, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:34, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178768}, {obj_id:35, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211414}, {obj_id:36, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224486}, {obj_id:41, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188622}, {obj_id:42, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199959}, {obj_id:43, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199095}, {obj_id:44, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199093}, {obj_id:45, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193076}, {obj_id:46, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214118}, {obj_id:47, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199546}, {obj_id:48, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199766}, {obj_id:49, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190029}, {obj_id:50, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186863}, {obj_id:51, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185594}, {obj_id:52, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200193}, {obj_id:53, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193157}, {obj_id:54, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188399}, {obj_id:55, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185761}, {obj_id:56, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:57, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185793}, {obj_id:58, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:59, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185738}, {obj_id:60, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192013}, {obj_id:61, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187261}, {obj_id:62, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183532}, {obj_id:63, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210345}, {obj_id:64, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183340}, {obj_id:65, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188170}, {obj_id:66, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183730}, {obj_id:67, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191888}, {obj_id:68, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183436}, {obj_id:69, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184586}, {obj_id:70, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188962}, {obj_id:71, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:308297}, {obj_id:72, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:251495}, {obj_id:73, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:304658}, {obj_id:74, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247858}, {obj_id:75, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201160}, {obj_id:77, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:81, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176269}, {obj_id:83, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:84, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:85, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:86, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:87, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:89, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:90, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:91, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:92, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:94, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183659}, {obj_id:97, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:100, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:103, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:106, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196176}, {obj_id:107, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188336}, {obj_id:108, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:275361}, {obj_id:109, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179806}, {obj_id:110, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191447}, {obj_id:111, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:275362}, {obj_id:112, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218570}, {obj_id:116, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:118, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:684299}, {obj_id:119, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:120, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190714}, {obj_id:121, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223442}, {obj_id:136, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177206}, {obj_id:137, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188473}, {obj_id:141, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238685}, {obj_id:148, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178773}, {obj_id:149, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174899}, {obj_id:151, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174973}, {obj_id:152, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174974}, {obj_id:153, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174899}, {obj_id:154, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176141}, {obj_id:155, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179296}, {obj_id:156, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178110}, {obj_id:158, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:159, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:160, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175011}, {obj_id:161, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175011}, {obj_id:162, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175061}, {obj_id:163, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175061}, {obj_id:164, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:165, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:167, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:168, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181212}, {obj_id:169, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177355}, {obj_id:170, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178587}, {obj_id:172, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177556}, {obj_id:173, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178796}, {obj_id:174, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:175, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176287}, {obj_id:176, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176601}, {obj_id:177, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:178, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177809}, {obj_id:179, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178207}, {obj_id:180, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179455}, {obj_id:181, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176359}, {obj_id:182, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177615}, {obj_id:183, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178110}, {obj_id:184, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176293}, {obj_id:185, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182472}, {obj_id:186, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181294}, {obj_id:187, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181212}, {obj_id:188, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178855}, {obj_id:189, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179247}, {obj_id:190, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180179}, {obj_id:191, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174869}, {obj_id:192, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176141}, {obj_id:193, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178955}, {obj_id:194, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179556}, {obj_id:195, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176751}, {obj_id:196, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182472}, {obj_id:197, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176028}, {obj_id:198, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176242}, {obj_id:199, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177698}, {obj_id:200, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177506}, {obj_id:201, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176601}, {obj_id:202, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178207}, {obj_id:203, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176359}, {obj_id:204, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179988}, {obj_id:205, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179191}, {obj_id:206, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:383555}, {obj_id:275, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:291, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176429}, {obj_id:301, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195808}, {obj_id:349, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:435, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:284362}, {obj_id:438, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277099}, {obj_id:441, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:365724}, {obj_id:442, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:352792}, {obj_id:477, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294946}, {obj_id:490, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210456}, {obj_id:501, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:326767}, {obj_id:503, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230400}, {obj_id:504, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185730}, {obj_id:506, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:12123}, {obj_id:507, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1069986}, {obj_id:508, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173197}, {obj_id:509, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199042}, {obj_id:510, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201819}, {obj_id:511, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197302}, {obj_id:513, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231048}, {obj_id:514, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222216}, {obj_id:515, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:516, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:519, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:520, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:340122}, {obj_id:635, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:281722}, {obj_id:636, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:313618}, {obj_id:637, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:404380}, {obj_id:638, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:513636}, {obj_id:642, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:406569}, {obj_id:643, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:526721}, {obj_id:644, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:324772}, {obj_id:645, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:385343}, {obj_id:652, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:336141}, {obj_id:653, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:407607}, {obj_id:656, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:302133}, {obj_id:657, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:349539}, {obj_id:718, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234380}, {obj_id:1036, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286678}, {obj_id:1093, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:316722}, {obj_id:1119, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:306729}, {obj_id:1131, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:226842}, {obj_id:1134, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1135, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204208}, {obj_id:1137, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1170, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1172, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1180, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1181, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1184, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:360170}, {obj_id:1205, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:226578}, {obj_id:1209, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234762}, {obj_id:1210, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175923}, {obj_id:1217, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185026}, {obj_id:1219, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186335}, {obj_id:1223, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186868}, {obj_id:1224, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187165}, {obj_id:1230, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196301}, {obj_id:1231, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:333432}, {obj_id:1242, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1250, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184636}, {obj_id:1277, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219147}, {obj_id:1280, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183351}, {obj_id:1281, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181880}, {obj_id:1282, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180188}, {obj_id:1283, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:343446}, {obj_id:1349, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212498}, {obj_id:1359, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200164}, {obj_id:1368, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:409123}, {obj_id:1371, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:527267}, {obj_id:1394, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212873}, {obj_id:1396, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194252}, {obj_id:1399, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194540}, {obj_id:1420, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:263353}, {obj_id:1421, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276973}, {obj_id:1428, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:349054}, {obj_id:1429, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:411748}, {obj_id:1432, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:439302}, {obj_id:1436, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276882}, {obj_id:1438, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298234}, {obj_id:1439, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:553609}, {obj_id:1441, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:425770}, {obj_id:1443, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:532331}, {obj_id:1444, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:441552}, {obj_id:1445, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:557166}, {obj_id:1446, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235923}, {obj_id:1447, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234251}, {obj_id:1449, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229611}, {obj_id:1471, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:249500}, {obj_id:1472, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303921}, {obj_id:1473, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:258588}, {obj_id:1475, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320137}, {obj_id:1480, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282443}, {obj_id:1481, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:362392}, {obj_id:1488, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286101}, {obj_id:1489, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:368900}, {obj_id:1493, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:274640}, {obj_id:1494, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294693}, {obj_id:1497, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:378392}, {obj_id:1498, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:457842}, {obj_id:1499, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:437052}, {obj_id:1501, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:428020}, {obj_id:1502, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:550068}, {obj_id:1503, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:412238}, {obj_id:1504, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:362578}, {obj_id:1505, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:535888}, {obj_id:1506, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:511046}, {obj_id:1507, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:433015}, {obj_id:1508, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:243979}, {obj_id:1509, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234595}, {obj_id:1510, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232619}, {obj_id:1511, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224335}, {obj_id:1512, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231595}, {obj_id:1513, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223657}, {obj_id:1514, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:346779}, {obj_id:1515, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231905}, {obj_id:1518, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:318041}, {obj_id:1519, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:310697}, {obj_id:1520, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:425065}, {obj_id:1521, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:367809}, {obj_id:1522, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:412013}, {obj_id:1523, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:509887}, {obj_id:1524, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:268389}, {obj_id:1527, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:336988}, {obj_id:1533, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:347500}, {obj_id:1535, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:353092}, {obj_id:1537, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:351196}, {obj_id:1540, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:477250}, {obj_id:1541, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:487084}, {obj_id:1542, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:483796}, {obj_id:1549, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203372}, {obj_id:1555, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:312593}, {obj_id:1556, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:415310}, {obj_id:1559, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207017}, {obj_id:1561, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208699}, {obj_id:1562, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205264}, {obj_id:1564, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216265}, {obj_id:1565, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:215597}, {obj_id:1566, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216631}, {obj_id:1567, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217307}, {obj_id:1569, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205718}, {obj_id:1571, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210530}, {obj_id:1577, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202860}, {obj_id:1578, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221503}, {obj_id:1579, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219106}, {obj_id:1582, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212516}, {obj_id:1586, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184380}, {obj_id:1588, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221044}, {obj_id:1590, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221684}, {obj_id:1591, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320059}, {obj_id:1599, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:1602, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:1619, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213180}, {obj_id:1642, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189353}, {obj_id:1643, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189220}, {obj_id:1644, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193462}, {obj_id:1645, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180107}, {obj_id:1654, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181394}, {obj_id:1657, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181674}, {obj_id:1658, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189842}, {obj_id:1666, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201840}, {obj_id:1668, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181740}, {obj_id:1669, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190036}, {obj_id:1670, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182184}, {obj_id:1708, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200183}, {obj_id:1709, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218850}, {obj_id:1717, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:319852}, {obj_id:1720, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:431890}, {obj_id:1726, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:563003}, {obj_id:1731, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:373531}, {obj_id:1746, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:296698}, {obj_id:1759, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:411595}, {obj_id:1760, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:521508}, {obj_id:1764, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260624}, {obj_id:1765, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:264190}, {obj_id:1766, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176744}, {obj_id:1767, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174432}, {obj_id:1768, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177085}, {obj_id:1769, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181110}, {obj_id:1770, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179110}, {obj_id:1771, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176843}, {obj_id:1773, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198614}, {obj_id:1776, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242568}, {obj_id:1777, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245745}, {obj_id:1778, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:249200}, {obj_id:1779, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:514775}, {obj_id:1780, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174650}, {obj_id:1781, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256438}, {obj_id:1782, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176308}, {obj_id:1783, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:269328}, {obj_id:1784, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241927}, {obj_id:1785, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195211}, {obj_id:1786, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1787, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1789, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:313384}, {obj_id:1790, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189174}, {obj_id:1794, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201926}, {obj_id:1795, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:263611}, {obj_id:1796, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:330941}, {obj_id:1800, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:252092}, {obj_id:1801, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1807, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214817}, {obj_id:1811, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223289}, {obj_id:1823, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320604}, {obj_id:1826, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:268234}, {obj_id:1827, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:369950}, {obj_id:1828, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286091}, {obj_id:1841, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297804}, {obj_id:1842, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:334896}, {obj_id:1848, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212552}, {obj_id:1850, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204050}, {obj_id:1858, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:239469}, {obj_id:1859, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213099}, {obj_id:1860, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:239048}, {obj_id:1868, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229019}, {obj_id:1869, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237507}, {obj_id:1871, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230088}, {obj_id:1872, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238192}, {obj_id:1881, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222260}, {obj_id:1882, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:255313}, {obj_id:1885, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203714}, {obj_id:1886, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205084}, {obj_id:1888, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225301}, {obj_id:1889, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229642}, {obj_id:1892, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237762}, {obj_id:1893, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187591}, {obj_id:1898, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204758}, {obj_id:1899, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213900}, {obj_id:1900, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202713}, {obj_id:1901, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204051}, {obj_id:1903, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204752}, {obj_id:1904, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213464}, {obj_id:1905, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189807}, {obj_id:1906, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189568}, {obj_id:1908, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177216}, {obj_id:1909, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189999}, {obj_id:1910, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190072}, {obj_id:1911, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184744}, {obj_id:1927, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:306481}, {obj_id:1928, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:404932}, {obj_id:1934, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218446}, {obj_id:1935, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:248651}, {obj_id:1939, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245943}, {obj_id:1940, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297508}, {obj_id:1941, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207520}, {obj_id:1942, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206244}, {obj_id:1943, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208189}, {obj_id:1944, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205856}, {obj_id:1945, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209158}, {obj_id:1946, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206074}, {obj_id:1947, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214746}, {obj_id:1948, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206759}, {obj_id:1953, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184677}, {obj_id:1961, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185383}, {obj_id:1962, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:338806}, {obj_id:1963, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:329717}, {obj_id:1964, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:400818}, {obj_id:1965, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:384947}, {obj_id:1966, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:215423}, {obj_id:1967, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193646}, {obj_id:1968, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180032}, {obj_id:1970, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207111}, {obj_id:1971, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207428}, {obj_id:1972, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216076}, {obj_id:1973, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207780}, {obj_id:1982, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191845}, {obj_id:1985, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177576}, {obj_id:1990, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177576}, {obj_id:1994, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186606}, {obj_id:1995, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233374}, {obj_id:1996, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208113}, {obj_id:1998, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231846}, {obj_id:1999, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227382}, {obj_id:2000, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189106}, {obj_id:2001, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197404}, {obj_id:2002, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189506}, {obj_id:2003, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187054}, {obj_id:2004, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188768}, {obj_id:2005, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241398}, {obj_id:2006, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188592}, {obj_id:2007, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185206}, {obj_id:2008, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189703}, {obj_id:2009, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199149}, {obj_id:2010, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276526}, {obj_id:2012, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276526}, {obj_id:2013, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187377}, {obj_id:2015, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189177}, {obj_id:2016, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191033}, {obj_id:2017, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192817}, {obj_id:2018, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203855}, {obj_id:2019, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196646}, {obj_id:2020, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199942}, {obj_id:2021, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203138}, {obj_id:2023, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193422}, {obj_id:2024, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222722}, {obj_id:2025, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194506}, {obj_id:2026, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184181}, {obj_id:2027, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191378}, {obj_id:2028, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206684}, {obj_id:2029, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203947}, {obj_id:2030, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206804}, {obj_id:2031, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204075}, {obj_id:2034, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194701}, {obj_id:2036, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206437}, {obj_id:2038, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196470}, {obj_id:2039, tenant_id:1003, log_del_time:922337 [2024-06-25 17:00:18.762704] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=476] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:18.765391] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.765401] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:18.765407] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.765414] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.765419] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.766200] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:18.766207] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:18.766210] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:18.766212] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:18.766216] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:18.766490] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=3] [APPLY STAT CB APPEND COST TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=32, this=0x2b3a9538a280, extra_info="") [2024-06-25 17:00:18.766499] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=9] [APPLY STAT CB IN QUEUE TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=267, this=0x2b3a9538a4b0, extra_info="") [2024-06-25 17:00:18.766502] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=3] [APPLY STAT CB WAIT COMMIT TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=0, this=0x2b3a9538a6e0, extra_info="") [2024-06-25 17:00:18.766506] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=4] [APPLY STAT CB EXECUTE TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=-521, this=0x2b3a9538a910, extra_info="") [2024-06-25 17:00:18.766511] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=4] [APPLY STAT CB TOTAL TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=-221, this=0x2b3a9538ab40, extra_info="") [2024-06-25 17:00:18.773864] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=9] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289107427328, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:18.775219] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.775228] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.775237] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.775244] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.775248] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.775660] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=1] fetch root ls success(local_cluster_id=1713323324, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.171:2882", sql_port:2881, role:1, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607828518, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.176:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:4, zone:"zone4", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607829360, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.189.192.96:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:5, zone:"zone5", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607830236, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.51:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607831113, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.56:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["10.186.69.171:2882", "10.191.69.51:2882", "10.191.69.56:2882", "10.186.69.176:2882", "10.189.192.96:2882"]) [2024-06-25 17:00:18.776403] WDIAG [RPC] send (ob_poc_rpc_proxy.h:156) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4012] sync rpc execute fail(ret=-4012, addr="10.191.69.52:2882", pcode=1315) [2024-06-25 17:00:18.776416] WDIAG [SQL.ENG] query_statistics (ob_px_tenant_target_monitor.cpp:275) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4012] send rpc to query statistics failed, need send refresh all(ret=-4012) [2024-06-25 17:00:18.776423] WDIAG [SQL.ENG] refresh_statistics (ob_px_tenant_target_monitor.cpp:114) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=6][errcode=-4012] query statistics failed(ret=-4012) [2024-06-25 17:00:18.776427] WDIAG [COMMON] operator() (ob_px_target_mgr.h:158) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] target monitor refresh statistics failed(ret=-4012, px_tenant_info={v:1002}, px_res_info->get_target_monitor()={is_init:true, tenant_id:1002, server:"10.191.69.51:2882", dummy_cache_leader:"10.191.69.52:2882", role:2}) [2024-06-25 17:00:18.777155] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62243][pnio1][T1001][YB420ABF4533-00061661718F652F-0-0] [lt=9][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:18.777780] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=48][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:18.777790] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:18.777795] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=999701, timeout_timestamp=1719306019777146) [2024-06-25 17:00:18.777802] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:18.777804] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:18.777809] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:18.777814] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:18.777817] WDIAG [SQL] open (ob_result_set.cpp:157) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:18.777820] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:18.777825] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.777830] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:18.777836] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=6][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:18.777843] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=6][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.777847] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.777850] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:18.777855] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:18.777860] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4013, ret=-4013) [2024-06-25 17:00:18.777865] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.777870] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=4][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:18.777873] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:18.777876] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=3][errcode=-4013] query failed(ret=-4013, conn=0x2b399eda8620, start=1719306018777407, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.777883] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=6][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:18.777886] WDIAG [SHARE.PT] get_by_tenant (ob_persistent_ls_table.cpp:639) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=2][errcode=-4013] execute sql failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.777904] WDIAG [SHARE.PT] get_by_tenant (ob_ls_table_operator.cpp:252) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=5][errcode=-4013] get all ls info by persistent_ls_ failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002) [2024-06-25 17:00:18.777909] WDIAG [SHARE.LOCATION] renew_location_for_tenant (ob_ls_location_service.cpp:806) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=5][errcode=-4013] fail to get all ls info(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, ls_infos=[]) [2024-06-25 17:00:18.777914] WDIAG [SHARE.LOCATION] renew_all_ls_locations (ob_ls_location_service.cpp:737) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=5][errcode=-4013] renew cache for tenant failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002) [2024-06-25 17:00:18.781363] WDIAG [SHARE.LOCATION] runTimerTask (ob_location_update_task.cpp:207) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF1-0-0] [lt=3][errcode=-4013] fail to renew_all_ls_locations(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.785003] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:18.786907] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.786917] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:18.786922] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.786929] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.786933] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.788679] INFO [STORAGE] update_id_meta (ob_ls_meta.cpp:681) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=10] update id meta(ret=0, service_type=0, limited_id=1719306035353935000, latest_scn={val:1719306018786724001, v:0}, *this={tenant_id:1, ls_id:{id:1}, ls_create_status:1, clog_checkpoint_scn:{val:1719272888853692000, v:0}, clog_base_lsn:{lsn:35095793664}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:1719285609440317000, v:0}, all_id_meta:{id_meta:[{limited_id:1719306035353935000, latest_log_ts:{val:1719306018786724001, v:0}}, {limited_id:80000001, latest_log_ts:{val:1719298896011036001, v:0}}, {limited_id:2660000001, latest_log_ts:{val:1719295281004326001, v:0}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}) [2024-06-25 17:00:18.788700] INFO [STORAGE.TRANS] handle_replay_result (ob_id_service.cpp:238) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=21] handle reply result(ret=0, service_type_=0, last_id=1719306035353935000, limited_id=1719306035353935000, log_ts={val:1719306018786724001, v:0}) [2024-06-25 17:00:18.788706] INFO [STORAGE.TRANS] replay (ob_id_service.cpp:211) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=5] ObIDService replay(service_type_=0, ret=0, ls_log={last_id_:1719306035353935000, limit_id_:1719306035353935000}, base_header={version:1, log_type:7, flag:0, need_pre_replay_barrier:false, need_post_replay_barrier:false, is_compressed:false, replay_hint:0}) [2024-06-25 17:00:18.793070] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.793079] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.793084] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.793091] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.793095] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.796778] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:18.796787] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.796793] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=3][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999958, timeout_timestamp=1719306048796415) [2024-06-25 17:00:18.796800] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:18.796802] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:18.796806] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:18.796810] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:18.796813] WDIAG [SQL] open (ob_result_set.cpp:157) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=2][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:18.796815] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:18.796819] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=3][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT * FROM __all_merge_info WHERE tenant_id = '1001'"}) [2024-06-25 17:00:18.796826] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=5][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT * FROM __all_merge_info WHERE tenant_id = '1001'"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:18.796833] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=7][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:18.796839] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=5][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.796843] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=4][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.796845] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:18.796850] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=3][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_merge_info WHERE tenant_id = '1001'"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:18.796856] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4A9-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_merge_info WHERE tenant_id = '1001'"}, aret=-4013, ret=-4013) [2024-06-25 17:00:18.796859] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT * FROM __all_merge_info WHERE tenant_id = '1001') [2024-06-25 17:00:18.796863] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:18.796867] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:18.796870] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] query failed(ret=-4013, conn=0x2b3b6150c330, start=1719306018796414, sql=SELECT * FROM __all_merge_info WHERE tenant_id = '1001') [2024-06-25 17:00:18.796875] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:18.796878] WDIAG [SHARE] load_global_merge_info (ob_global_merge_table_operator.cpp:49) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=2][errcode=-4013] fail to execute sql(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1001, meta_tenant_id=1001, sql=SELECT * FROM __all_merge_info WHERE tenant_id = '1001') [2024-06-25 17:00:18.796901] WDIAG [STORAGE] refresh_merge_info (ob_tenant_freeze_info_mgr.cpp:890) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=6][errcode=-4013] failed to load global merge info(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", global_merge_info={tenant_id:1001, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}}) [2024-06-25 17:00:18.796922] WDIAG [STORAGE] runTimerTask (ob_tenant_freeze_info_mgr.cpp:1005) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=20][errcode=-4013] fail to refresh merge info(tmp_ret=-4013, tmp_ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.797586] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:18.797593] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.797598] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=3][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=1999716, timeout_timestamp=1719306020796926) [2024-06-25 17:00:18.797604] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:18.797606] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:18.797609] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=3][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:18.797615] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=5][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:18.797617] WDIAG [SQL] open (ob_result_set.cpp:157) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=2][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:18.797619] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:18.797622] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=3][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}) [2024-06-25 17:00:18.797627] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=5][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:18.797631] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=4][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:18.797638] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=6][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.797640] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=2][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:18.797645] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=5][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:18.797649] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=2][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:18.797653] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AA-0-0] [lt=4][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, aret=-4013, ret=-4013) [2024-06-25 17:00:18.797656] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.797660] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:18.797664] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:18.797667] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=2][errcode=-4013] query failed(ret=-4013, conn=0x2b3a78d44050, start=1719306018797176, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.797671] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:18.797676] WDIAG [SHARE] load (ob_core_table_proxy.cpp:436) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] execute sql failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1001, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.797691] WDIAG [SHARE] load (ob_core_table_proxy.cpp:368) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] load failed(ret=-4013, for_update=false) [2024-06-25 17:00:18.797696] WDIAG [SHARE] get (ob_global_stat_proxy.cpp:442) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] core_table load failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.797699] WDIAG [SHARE] get_snapshot_gc_scn (ob_global_stat_proxy.cpp:164) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] get failed(ret=-4013) [2024-06-25 17:00:18.797703] WDIAG [STORAGE] get_global_info (ob_tenant_freeze_info_mgr.cpp:811) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] fail to get global info(ret=-4013, tenant_id=1001) [2024-06-25 17:00:18.797708] WDIAG [STORAGE] try_update_info (ob_tenant_freeze_info_mgr.cpp:954) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] failed to get global info(ret=-4013) [2024-06-25 17:00:18.797712] WDIAG [STORAGE] runTimerTask (ob_tenant_freeze_info_mgr.cpp:1008) [63430][T1001_FreInfoRe][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] fail to try update info(tmp_ret=-4013, tmp_ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.802954] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=51000, remain_us=25924222, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=51, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:18.807546] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.807555] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.807561] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:18.807567] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.807574] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.810305] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95515][T1_L0_G1][T1][YB420ABA45B1-00061645BDA93764-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=12, avg_log_batch_cnt=0, total_group_log_size=4682, avg_group_log_size=390) [2024-06-25 17:00:18.810347] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.810356] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.810360] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:18.810366] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.810371] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.810378] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=14] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=4682) [2024-06-25 17:00:18.811192] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62745][T1_ReplaySrv2][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:18.811294] WDIAG [RPC] send (ob_poc_rpc_proxy.h:156) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=3][errcode=-4012] sync rpc execute fail(ret=-4012, addr="10.191.69.52:2882", pcode=1308) [2024-06-25 17:00:18.811308] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=9][errcode=-4012] rpc task_execute fail(ret=-4012, tenant_id=1002, svr="10.191.69.52:2882", timeout=1999699, timeout_timestamp=1719306018808142) [2024-06-25 17:00:18.811315] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=6][errcode=-4012] task execute failed(ret=-4012) [2024-06-25 17:00:18.811319] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=3][errcode=-4012] execute with sql failed(ret=-4012) [2024-06-25 17:00:18.811324] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=4][errcode=-4012] fail execute plan(ret=-4012) [2024-06-25 17:00:18.811327] WDIAG [SQL] open (ob_result_set.cpp:157) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=2][errcode=-4012] execute plan failed(ret=-4012) [2024-06-25 17:00:18.811330] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=2][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:18.811333] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=2][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}) [2024-06-25 17:00:18.811339] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=5][errcode=-4012] execute failed(ret=-4012, tenant_id=1002, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1718209168713032) [2024-06-25 17:00:18.811344] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=4][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:18.811352] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=6][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:18.811355] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=3][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:18.811358] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=3][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:18.811366] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=4][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:18.811371] INFO [SERVER] process_final (ob_inner_sql_connection.cpp:734) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=5] slow inner sql(last_ret=-4012, sql={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, process_time=2002967) [2024-06-25 17:00:18.811375] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8642-0-0] [lt=4][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, aret=-4012, ret=-4012) [2024-06-25 17:00:18.811380] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1002, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.811384] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1002) [2024-06-25 17:00:18.811387] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1002) [2024-06-25 17:00:18.811391] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] query failed(ret=-4012, conn=0x2b3a78d45a30, start=1719306016808398, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.811396] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] read failed(ret=-4012) [2024-06-25 17:00:18.811399] WDIAG [SHARE] load (ob_core_table_proxy.cpp:436) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1002, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:18.811431] WDIAG [SHARE] load (ob_core_table_proxy.cpp:368) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] load failed(ret=-4012, for_update=false) [2024-06-25 17:00:18.811436] WDIAG [SHARE] get (ob_global_stat_proxy.cpp:442) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] core_table load failed(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:18.811438] WDIAG [SHARE] get_snapshot_gc_scn (ob_global_stat_proxy.cpp:164) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=2][errcode=-4012] get failed(ret=-4012) [2024-06-25 17:00:18.811441] WDIAG [STORAGE] get_global_info (ob_tenant_freeze_info_mgr.cpp:811) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] fail to get global info(ret=-4012, tenant_id=1002) [2024-06-25 17:00:18.811446] WDIAG [STORAGE] try_update_info (ob_tenant_freeze_info_mgr.cpp:954) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] failed to get global info(ret=-4012) [2024-06-25 17:00:18.811448] WDIAG [STORAGE] runTimerTask (ob_tenant_freeze_info_mgr.cpp:1008) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=2][errcode=-4012] fail to try update info(tmp_ret=-4012, tmp_ret="OB_TIMEOUT") [2024-06-25 17:00:18.814142] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.814150] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.814154] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:18.814160] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.814164] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.819924] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.819931] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.819935] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.819940] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.819944] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.821253] INFO do_work (ob_rl_mgr.cpp:709) [62262][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=9] swc wakeup.(stat_period_=1000000, ready=false) [2024-06-25 17:00:18.822262] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=76000, remain_us=23002717, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=76, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:18.822508] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=27] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:18.825065] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABF4533-0006166171EF4863-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301543, send_timestamp=1719305996523521) [2024-06-25 17:00:18.826921] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.826930] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.826935] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.826941] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.826946] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.829583] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:18.829616] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:18.829621] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:18.829623] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:18.829626] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:18.829664] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1039, self="10.191.69.51:2882", committed size=1211) [2024-06-25 17:00:18.829766] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.829774] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.829779] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.829784] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.829790] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.829816] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325962AAE-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305996528272) [2024-06-25 17:00:18.829854] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AAE-0-0] [lt=2] [PALF STAT LOG TASK TIME](palf_id=1039, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=2, avg_log_gen_to_submit_time=28, avg_log_submit_to_flush_time=87, avg_log_submit_to_slide_time=286) [2024-06-25 17:00:18.829887] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AAF-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1005, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=20, avg_log_submit_to_flush_time=92, avg_log_submit_to_slide_time=275) [2024-06-25 17:00:18.832411] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:18.833389] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF59BB-0-0] [lt=4] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1333, avg_group_log_size=121) [2024-06-25 17:00:18.837918] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:18.839508] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.839516] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=8][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.839522] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.839527] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.839530] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.839535] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.839539] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.839545] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.839551] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.839555] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.839557] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=1][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.839562] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.839568] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.839571] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.839574] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.839577] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.839582] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] query failed(ret=-5066, conn=0x2b3a78d45750, start=1719306018838045, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.839587] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.839609] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.839614] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.839620] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.839626] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.839630] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.839635] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.839642] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.839648] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.839652] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.839662] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=0] will sleep(sleep_us=33000, remain_us=360477, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=33, v.err_=-4383, timeout_timestamp=1719306019200138) [2024-06-25 17:00:18.840655] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C7F23FFD2B-0-0] [lt=6] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1005, child={server:"10.186.69.171:2882", region:"sys_region", register_time_us:1719306018839797, keepalive_ts:-1}) [2024-06-25 17:00:18.840661] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C7F23FFD2B-0-0] [lt=6] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1005, self="10.191.69.51:2882", child={server:"10.186.69.171:2882", region:"sys_region", register_time_us:1719306018839797, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:18.840923] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=0][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:18.840935] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=0] will sleep(sleep_us=13000, remain_us=1710975, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=13, v.err_=-4383, timeout_timestamp=1719306020551910) [2024-06-25 17:00:18.844925] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABF4533-0006166172BEBE86-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301554, send_timestamp=1719305996543370) [2024-06-25 17:00:18.847570] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.847578] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.847582] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.847588] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.847593] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.849737] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A07F-0-0] [lt=11] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a5d1db6a0, extra_info="palf_id:1") [2024-06-25 17:00:18.849746] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A07F-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=829, this=0x2b3a5d1db8d0, extra_info="palf_id:1") [2024-06-25 17:00:18.849908] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1333) [2024-06-25 17:00:18.855437] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AB1-0-0] [lt=6] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=89, avg_log_submit_to_slide_time=839) [2024-06-25 17:00:18.855444] INFO [PALF] read (log_cache.cpp:100) [99517][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=3] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:18.856177] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=10093091, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=162, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:18.857297] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5788043, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=190, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:18.859913] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325C300C7-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301543, send_timestamp=1719305996558369) [2024-06-25 17:00:18.860212] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABF4538-00061669548FCBEC-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301547, send_timestamp=1719305996558664) [2024-06-25 17:00:18.860240] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABF4538-00061669548FCBEC-0-0] [lt=9] [PALF STAT LOG TASK TIME](palf_id=1010, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=93, avg_log_submit_to_slide_time=292) [2024-06-25 17:00:18.860247] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1010, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:18.861943] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B2D-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1009, self="10.191.69.51:2882", committed size=1098) [2024-06-25 17:00:18.862021] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1098) [2024-06-25 17:00:18.866259] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:18.866266] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:18.866268] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:18.866271] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:18.866274] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:18.866501] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-000616432547EFEF-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305996564957) [2024-06-25 17:00:18.873288] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.873296] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:18.873300] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:18.873305] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.873310] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.874048] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-0006164324713060-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=22301542, send_timestamp=1719305996572505) [2024-06-25 17:00:18.875817] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.875825] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.875829] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.875832] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.875836] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.875840] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.875844] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.875849] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.875855] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.875858] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.875861] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.875867] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.875871] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.875876] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.875879] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.875882] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.875887] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] query failed(ret=-5066, conn=0x2b3a78d44610, start=1719306018874372, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.875892] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.875912] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.875917] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.875922] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.875927] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.875930] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.875933] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.875939] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.875943] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.875949] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.877215] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62236][pnio1][T1001][YB420ABF4533-00061661718F6530-0-0] [lt=9][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:18.877528] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:18.877538] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:18.877544] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=4][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999956, timeout_timestamp=1719306048877079) [2024-06-25 17:00:18.877551] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:18.877553] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:18.877557] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=3][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:18.877561] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:18.877563] WDIAG [SQL] open (ob_result_set.cpp:157) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE857-0-0] [lt=2][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:18.879736] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.879743] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.879748] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.879751] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.879754] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.879757] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.879763] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.879768] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.879775] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.879777] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.879779] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.879785] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.879790] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.879793] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.879798] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.879801] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.879804] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] query failed(ret=-5066, conn=0x2b3a78d44330, start=1719306018878005, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.879810] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.879829] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.879834] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.879839] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.879844] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.879848] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.879852] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.879858] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.879863] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.879868] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.885059] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:18.885352] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.885359] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.885364] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.885366] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.885369] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.885373] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.885378] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.885383] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.885390] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.885394] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.885397] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.885402] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.885407] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.885411] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.885415] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.885418] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.885421] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] query failed(ret=-5066, conn=0x2b3afefeaeb0, start=1719306018883929, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.885427] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.885444] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.885449] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.885454] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.885459] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.885462] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.885469] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.885475] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.885479] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.885485] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.888317] INFO pktc_resp_cb_on_msg (pktc_resp.h:70) [62247][pnio1][T0][Y0-0000000000000000-0-0] [lt=8] PNIO resp cb not found: packet_id=1035875065 [2024-06-25 17:00:18.892862] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.894921] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.894928] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.894932] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.894935] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.894938] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.894943] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.894947] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.894953] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.894958] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.894961] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.894964] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.894969] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.894974] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.894978] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.894981] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.894986] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.894989] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] query failed(ret=-5066, conn=0x2b3b6150ceb0, start=1719306018893545, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.894993] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.895012] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.895016] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.895020] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.895025] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.895028] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.895032] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.895038] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.895041] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.895047] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.896067] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100009][T1004_L0_G0][T1004][YB420ABF4534-000616451CB7B7D9-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:18.897616] WDIAG [SERVER] do_heartbeat_event (ob_heartbeat.cpp:265) [62788][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] try reload schema failed(schema_version=1719272886792320, refresh_schema_info={schema_version:1719272886792320, tenant_id:1, sequence_id:1099511641796}, schema_ret=-4023) [2024-06-25 17:00:18.898431] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2710363, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=210, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:18.899397] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=23][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-06-25 17:00:18.899406] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=8] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-06-25 17:00:18.899425] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=8] get_max_applied_scn(scn={val:1713323697227896000, v:0}, this={ls_id_:{id:1007}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713323697227896000, v:0}}) [2024-06-25 17:00:18.899435] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=8] get_min_unreplayed_log_info(lsn={lsn:85187186570}, scn={val:1719306018370414005, v:0}, this={ls_id_:{id:1007}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306018850095, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:85187186570}, next_to_submit_scn_:{val:1719306018370414005, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713323697227896000, v:0}, iterator_:{iterator_impl:{buf_:0x2b4185039000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:85187186448}, end_lsn:{lsn:85187186570}, read_buf:{buf_len_:86016, buf_:0x2b4185039000}, block_size:67104768, log_storage_:0x2b3a86d23470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306018370414004, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306018370414004, v:0}, data_checksum:2442502576, flag:1}}, init_mode_version:0, accumulate_checksum:2847000297, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306018370414004, v:0}}}}}) [2024-06-25 17:00:18.900525] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100226][T1004_L0_G0][T1004][YB420ABF4534-000616451CB7B7D9-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:18.900775] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=14834048, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=131, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:18.901097] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=4] re_register_parent reason(palf_id=1009, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306014798883, last_first_register_time_us=1713323716483453, register_time_us=1719306014798883, curr_time_us=1719306018901097) [2024-06-25 17:00:18.901120] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=11] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1009, self="10.191.69.51:2882", reason=1, leader="10.186.69.176:2882", register_time_us=1719306018901109, last_submit_register_req_time_us=1719306018901109, register_parent_reason=1) [2024-06-25 17:00:18.901127] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=7] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1009, self="10.191.69.51:2882") [2024-06-25 17:00:18.903574] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2072317, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=214, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:18.907893] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=52000, remain_us=25819282, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=52, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:18.910246] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [62585][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=13] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-06-25 17:00:18.912479] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:18.912485] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:18.912490] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:18.912492] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:18.912496] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:18.912500] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:18.912504] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:18.912510] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:18.912516] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.912520] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:18.912522] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:18.912526] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:18.912531] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:18.912536] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.912539] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:18.912542] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:18.912546] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] query failed(ret=-5066, conn=0x2b399eda8060, start=1719306018911109, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:18.912551] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:18.912569] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:18.912574] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.912579] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:18.912584] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.912587] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:18.912590] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:18.912596] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:4000000}, GTI_LS={id:1}) [2024-06-25 17:00:18.912600] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.912606] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:18.914326] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [100337][T1003_Occam][T1003][Y0-0000000000000000-0-0] [lt=13] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-06-25 17:00:18.914596] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3790385, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=203, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:18.916811] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:18.916820] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:18.916825] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:18.916832] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.916838] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:18.922638] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [62718][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=12] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=969588866, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=408742, mem_used=767425536) [2024-06-25 17:00:18.929638] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:18.929673] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT GROUP LOG INFO](palf_id=1005, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:18.929682] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:18.929686] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:18.929688] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:18.929691] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:18.929912] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164325962AB4-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301546, send_timestamp=1719305996628365) [2024-06-25 17:00:18.931095] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-000616430B73EF50-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=330, fly_ts=22301542, send_timestamp=1719305996629553) [2024-06-25 17:00:18.931153] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=2) [2024-06-25 17:00:18.932467] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:18.933468] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1211) [2024-06-25 17:00:18.937976] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:18.938306] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [63346][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=12] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-06-25 17:00:18.939409] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [42764][T1002_ComTimerT][T1002][Y0-0000000000000000-0-0] [lt=9] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-06-25 17:00:18.940594] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1A-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:18.945321] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [63469][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=4] ====== checkpoint timer task ====== [2024-06-25 17:00:18.945332] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:193) [63469][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=8] no logstream(ret=0, ls_cnt=0) [2024-06-25 17:00:18.945488] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:18.948088] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [100333][T1003_Occam][T1003][YB420ABF4533-000616615EBF73AC-0-0] [lt=9] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:18.952106] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=77000, remain_us=22872872, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=77, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:18.955422] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [62818][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=7] current monitor number(seq_=-1) [2024-06-25 17:00:18.958456] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] ====== checkpoint timer task ====== [2024-06-25 17:00:18.958938] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] succ to get rec scn(*this={this:0x2b3aae604030, ls_id:{id:1006}, tenant_id:1002, state:"L_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171811467, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:151}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.958954] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=16] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb4080, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171812500, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb4080, snapshot_version:{val:1718209171812500, v:0}, ls_id:{id:1006}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.958974] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=15] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1006}) [2024-06-25 17:00:18.958997] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=9] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3a9534a510, is_inited:true, freezing_scn:{val:1718285174571744002, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1006}, ls_create_status:1, clog_checkpoint_scn:{val:1719306013505123002, v:0}, clog_base_lsn:{lsn:75425759232}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315392841000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:1, proposal_id:1, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1006}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1006, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:110, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:18.959030] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=31] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1006}) [2024-06-25 17:00:18.959045] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] get_binary_search_range_(ret=0, this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1112, max_block_id=1124, result_block_id=274894685184, committed_lsn={lsn:75469139099}, scn={val:1719306018752321001, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:18.959587] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=12] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759415, offset=21479848}, end_cursor:ObLogCursor{file_id=67, log_id=18759416, offset=21480089}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb8c050, len:3672, log_data_len:241, seq:18759415, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:18.959605] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:75425759232}, checkpoint_scn={val:1719306018752321001, v:0}, ls_id={id:1006}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:18.959617] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:75425759232}, new_base_lsn={lsn:75425759232}, lsn={lsn:75425759232}) [2024-06-25 17:00:18.959625] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:75425759232}, ls->get_ls_id()={id:1006}) [2024-06-25 17:00:18.959958] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325C300CB-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301544, send_timestamp=1719305996658414) [2024-06-25 17:00:18.959982] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C300CB-0-0] [lt=5] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf16f470, extra_info="") [2024-06-25 17:00:18.960081] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=79, this=0x2b3aaf1793b0, extra_info="palf_id:1010") [2024-06-25 17:00:18.960095] INFO [LIB] stat (utility.h:1214) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=5, this=0x2b3a8ce1d3b0, extra_info="") [2024-06-25 17:00:18.960094] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] succ to get rec scn(*this={this:0x2b3aae6b6030, ls_id:{id:1007}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171811972, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:149}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.960102] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb4e90, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171813010, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb4e90, snapshot_version:{val:1718209171813010, v:0}, ls_id:{id:1007}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.960106] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=6, this=0x2b3aaf1795e0, extra_info="palf_id:1010") [2024-06-25 17:00:18.960111] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1007}) [2024-06-25 17:00:18.960130] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3a86ce4510, is_inited:true, freezing_scn:{val:1718289975940695002, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1007}, ls_create_status:1, clog_checkpoint_scn:{val:1719306013868334001, v:0}, clog_base_lsn:{lsn:85155950592}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315408583001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1007}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1007, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:18.960154] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=23] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1007}) [2024-06-25 17:00:18.960161] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] get_binary_search_range_(ret=0, this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1257, max_block_id=1269, result_block_id=274894685184, committed_lsn={lsn:85187186692}, scn={val:1719306018870127000, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:18.960287] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC8D5-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf16f6a0, extra_info="palf_id:1010") [2024-06-25 17:00:18.960292] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC8D5-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=295, this=0x2b3aaf16f8d0, extra_info="palf_id:1010") [2024-06-25 17:00:18.960635] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=12] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759416, offset=21480089}, end_cursor:ObLogCursor{file_id=67, log_id=18759417, offset=21480330}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb98050, len:3431, log_data_len:241, seq:18759416, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:18.960646] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:85155950592}, checkpoint_scn={val:1719306018870127000, v:0}, ls_id={id:1007}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:18.960653] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:85155950592}, new_base_lsn={lsn:85155950592}, lsn={lsn:85155950592}) [2024-06-25 17:00:18.960659] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:85155950592}, ls->get_ls_id()={id:1007}) [2024-06-25 17:00:18.961092] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62241][pnio1][T0][YB420ABF4533-0006166171AF65D4-0-0] [lt=7][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:18.961103] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62241][pnio1][T0][YB420ABF4533-0006166171AF65D4-0-0] [lt=11] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:18.961120] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62241][pnio1][T0][YB420ABF4533-0006166171AF65D4-0-0] [lt=8] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306018960742]}) [2024-06-25 17:00:18.961126] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=3] succ to get rec scn(*this={this:0x2b3ab7604030, ls_id:{id:1008}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629001, v:0}, last_push_gc_task_ts_:1718209171812477, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:149}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.961134] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb5ca0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171815569, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb5ca0, snapshot_version:{val:1718209171815569, v:0}, ls_id:{id:1008}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.961144] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=9] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1008}) [2024-06-25 17:00:18.961164] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3a86daa510, is_inited:true, freezing_scn:{val:1718286006014126001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1008}, ls_create_status:1, clog_checkpoint_scn:{val:1719306013537772003, v:0}, clog_base_lsn:{lsn:74083663872}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315055094004, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1008}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1008, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-550728704, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:18.961186] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=22] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1008}) [2024-06-25 17:00:18.961192] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] get_binary_search_range_(ret=0, this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1092, max_block_id=1104, result_block_id=274894685184, committed_lsn={lsn:74143917071}, scn={val:1719306018543922003, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:18.961690] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759417, offset=21480330}, end_cursor:ObLogCursor{file_id=67, log_id=18759418, offset=21480571}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb7c050, len:3190, log_data_len:241, seq:18759417, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:18.961702] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:74083663872}, checkpoint_scn={val:1719306018543922003, v:0}, ls_id={id:1008}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:18.961716] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:74083663872}, new_base_lsn={lsn:74083663872}, lsn={lsn:74083663872}) [2024-06-25 17:00:18.961724] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:74083663872}, ls->get_ls_id()={id:1008}) [2024-06-25 17:00:18.962197] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] succ to get rec scn(*this={this:0x2b3ab76b6030, ls_id:{id:1009}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171812974, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:152}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.962207] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb6ab0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171817146, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb6ab0, snapshot_version:{val:1718209171817146, v:0}, ls_id:{id:1009}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.962221] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=11] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1009}) [2024-06-25 17:00:18.962247] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3aaf086510, is_inited:true, freezing_scn:{val:1718283438009682001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1009}, ls_create_status:1, clog_checkpoint_scn:{val:1719306013750401001, v:0}, clog_base_lsn:{lsn:86632255488}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315224596003, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1009}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1009, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:18.962276] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=27] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1009}) [2024-06-25 17:00:18.962286] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] get_binary_search_range_(ret=0, this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1279, max_block_id=1291, result_block_id=274894685184, committed_lsn={lsn:86651599708}, scn={val:1719306018805948000, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:18.962762] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759418, offset=21480571}, end_cursor:ObLogCursor{file_id=67, log_id=18759419, offset=21480812}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb88050, len:2949, log_data_len:241, seq:18759418, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:18.962772] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:86632255488}, checkpoint_scn={val:1719306018805948000, v:0}, ls_id={id:1009}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:18.962780] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:86632255488}, new_base_lsn={lsn:86632255488}, lsn={lsn:86632255488}) [2024-06-25 17:00:18.962786] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:86632255488}, ls->get_ls_id()={id:1009}) [2024-06-25 17:00:18.963257] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=3] succ to get rec scn(*this={this:0x2b3a84404030, ls_id:{id:1010}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171813475, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:151}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.963265] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb78c0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171818018, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb78c0, snapshot_version:{val:1718209171818018, v:0}, ls_id:{id:1010}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:18.963276] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=9] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1010}) [2024-06-25 17:00:18.963294] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3aaf132510, is_inited:true, freezing_scn:{val:1718283437110618003, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1010}, ls_create_status:1, clog_checkpoint_scn:{val:1719198894656674000, v:0}, clog_base_lsn:{lsn:75627073536}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315483366000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1010}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1010, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:18.963319] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=23] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x2b3aaf1321b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x2b3aaf137cd0, table_type:1, memtable_head:1774, memtable_tail:1775, t3m:0x2b3a8a6b2030, tables:[null, null, null, null, null, null, null, null, null, null, null, null, null, null, 0x2b3a86c90080, null]}, is_freezing:false, ls_id:{id:1010}, mini_merge_recycle_commit_versions_ts:1719188582135344, tx_data_table:0x2b3aaf1391d0, ls_tablet_svr:0x2b3aaf132190, slice_allocator:0x2b3aaf139210}, min_rec_scn={val:1719198894656674000, v:0}, ls_id_={id:1010}) [2024-06-25 17:00:18.963334] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=13] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1719198894656674000, v:0}, checkpoint_scn_in_ls_meta={val:1719198894656674000, v:0}, ls_id={id:1010}, service_type="TRANS_SERVICE") [2024-06-25 17:00:18.963341] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1010, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:75627073536}, new_base_lsn={lsn:75627073536}, lsn={lsn:75627073536}) [2024-06-25 17:00:18.963346] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:75627073536}, ls->get_ls_id()={id:1010}) [2024-06-25 17:00:18.963350] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:191) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] succeed to update_clog_checkpoint(ret=0, ls_cnt=5) [2024-06-25 17:00:18.966211] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [62616][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=7] ====== tenant freeze timer task ====== [2024-06-25 17:00:18.966276] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:18.966286] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:18.966289] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:18.966291] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:18.966294] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:18.966514] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-000616432547EFF3-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301556, send_timestamp=1719305996664957) [2024-06-25 17:00:18.974085] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=12] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289109524480, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:18.975028] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [99158][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=29] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}, {1001,0,False,{False,},False,False,True}, {1002,1,False,{False,},False,False,True}, {1003,1,False,{False,},False,False,True}, {1004,1,False,{False,},False,False,True}, {1005,0,False,{False,},False,False,True}, {1006,1,False,{False,},False,False,True}, {1007,1,False,{False,},False,False,True}, {1008,1,False,{False,},False,False,True}, {1039,0,False,{False,},False,False,True}]) [2024-06-25 17:00:18.977251] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62244][pnio1][T1001][YB420ABF4533-00061661718F6531-0-0] [lt=10][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:18.985113] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.002097] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SYS](tenant_id=1, io_status=sys_group_name: MICRO_BLOCK_CACHE_IO, mode: read, size: 4096.00, iops: 11.19, rt: 76.00) [2024-06-25 17:00:19.002108] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS CONFIG](tenant_id=1, ref_cnt=2, io_config={group_num_:0, memory_limit_:8589934592, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:4}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=12763552, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:19.002142] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=15] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 0.93, rt: 105.00) [2024-06-25 17:00:19.002148] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SYS](tenant_id=500, io_status=sys_group_name: INSPECT_BAD_BLOCK_IO, mode: read, size: 2097152.00, iops: 0.93, rt: 775.00) [2024-06-25 17:00:19.002152] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:19.002188] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.73, rt: 68.25) [2024-06-25 17:00:19.002193] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:1073741824, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:1}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10735904, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:19.002219] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SYS](tenant_id=1004, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 4.66, rt: 66.80) [2024-06-25 17:00:19.002224] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=1004, ref_cnt=2, io_config={group_num_:0, memory_limit_:724775731200, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:100}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=12763552, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:19.002236] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002244] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002246] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002250] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002255] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002257] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002261] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002264] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002268] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002271] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002274] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002278] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002281] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002284] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002288] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.002291] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:19.009756] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=9939511, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=163, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.010125] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_receive_op.cpp:80) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] check physical plan status failed(ret=-5066) [2024-06-25 17:00:19.010134] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=8][errcode=-5066] inner get next row failed(ret=-5066, type=15, op="PHY_DIRECT_RECEIVE", op_id=0) [2024-06-25 17:00:19.010143] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=9][errcode=-5066] get next row from operator failed(ret=-5066) [2024-06-25 17:00:19.010146] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] get next row from exec result failed(ret=-5066) [2024-06-25 17:00:19.010150] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get_next_row failed(ret=-5066, has_tenant_resource()=true) [2024-06-25 17:00:19.010155] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] result set open failed(ret=-5066, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.010160] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute failed(ret=-5066, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:19.010166] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-5066, err_:"OB_ERR_SESSION_INTERRUPTED", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:19.010174] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:19.010178] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] result set close failed(ret=-5066) [2024-06-25 17:00:19.010180] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to close result(close_ret=-5066, ret=-5066) [2024-06-25 17:00:19.010186] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-5066, ret=-5066) [2024-06-25 17:00:19.010192] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-5066, ret=-5066) [2024-06-25 17:00:19.010195] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] execute sql failed(ret=-5066, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.010201] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] retry_while_no_tenant_resource failed(ret=-5066, tenant_id=1) [2024-06-25 17:00:19.010204] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] execute_read failed(ret=-5066, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:19.010208] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] query failed(ret=-5066, conn=0x2b3b6150d470, start=1719306019008713, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:19.010215] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] execute sql failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.010240] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] get all ls info by persistent_ls_ failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", tenant_id=1001) [2024-06-25 17:00:19.010245] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1250) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fail to batch get ls info by operator(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.010251] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-5066] batch renew ls locations failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.010257] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] renew location failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.010261] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get location(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.010264] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-5066] fail to get log stream location leader(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.010271] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-5066] get leader failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.010276] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5] fetch new DAS ID range finish(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.010282] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] fetch new range failed(ret=-5066, ret="OB_ERR_SESSION_INTERRUPTED") [2024-06-25 17:00:19.010955] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5634385, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=191, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.013808] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=53000, remain_us=25713368, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=53, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.020588] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.020598] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:19.020604] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:19.020611] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.020616] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.022811] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.022824] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.022843] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=1] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e6b6780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221534415, assoc_session_id:3221534415, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306018751663, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3a97f681f8, id:3221534415, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e6b6780}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:19.025313] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.025322] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.025327] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:19.025333] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.025337] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.025962] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AB7-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=97, avg_log_submit_to_slide_time=845) [2024-06-25 17:00:19.025971] INFO [PALF] read (log_cache.cpp:100) [99518][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.027404] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63474][T1001_LSArchive][T1001][YB420ABF4533-000616423F8E6725-0-0] [lt=9] gc stale ls task succ [2024-06-25 17:00:19.029711] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT GROUP LOG INFO](palf_id=1039, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:19.029720] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99931][T1004_L0_G0][T1004][YB420ABA45B1-000616460A03D1DF-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.029724] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=12] [PALF STAT APPEND DATA SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1211) [2024-06-25 17:00:19.029734] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.029767] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT APPEND DATA SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1211) [2024-06-25 17:00:19.029773] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.029776] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.029779] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.029783] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.029841] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1333) [2024-06-25 17:00:19.029955] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABDC060-0006164325962AB8-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301540, send_timestamp=1719305996728415) [2024-06-25 17:00:19.029993] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325962AB9-0-0] [lt=0][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301538, send_timestamp=1719305996728454) [2024-06-25 17:00:19.031180] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.031189] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.031193] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.031199] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.031204] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.032370] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4383, dropped:1103, tid:62275}]) [2024-06-25 17:00:19.032493] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.032930] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [100318][T1004_L0_G0][T1004][YB420ABF4533-00061660EF5FD0A2-0-0] [lt=8] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1719305953432726, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1719306019032157, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-06-25 17:00:19.032982] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [100318][T1004_L0_G0][T1004][YB420ABF4533-00061660EF5FD0A2-0-0] [lt=13] current clock is (clock_=0, max_seq_=0, mem_limit=161956757504, cur_mem_hold=55475961856, attr_.tenant_id_=1004) [2024-06-25 17:00:19.034130] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100263][T1004_L0_G0][T1004][YB420ABA45B1-000616460A03D1DF-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.038032] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.041050] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.041058] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.041062] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.041068] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.041072] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.050141] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFF5-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1333) [2024-06-25 17:00:19.052327] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.052337] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.052341] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78dd0d80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306018997478, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.052359] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78dd0d80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306018997478, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.052371] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.052377] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.052380] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.052384] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.052386] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.052389] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.052393] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=211, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.052398] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=2556396, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=211, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.052799] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [99392][T1004_ReqMemEvi][T1004][Y0-0000000000000000-0-0] [lt=11] sql audit evict task end(request_manager_->get_tenant_id()=1004, evict_high_mem_level=20669530113, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=8345564, mem_used=15148855296) [2024-06-25 17:00:19.053460] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63292][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=8] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7268, mem_used=12478464) [2024-06-25 17:00:19.054680] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.054689] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.054692] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a69360780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019000641, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.054709] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a69360780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019000641, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.054720] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.054725] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.054728] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.054731] WDIAG [SQL] open (ob_result_set.cpp:157) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.054733] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.054737] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.054741] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=132, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.054746] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4] will sleep(sleep_us=100000, remain_us=14680077, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=132, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.054772] INFO [LIB] stat (utility.h:1214) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbfec70, extra_info="") [2024-06-25 17:00:19.057241] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.057252] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.057256] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb219e9e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019002748, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.057279] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=23][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb219e9e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019002748, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.057291] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.057299] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.057304] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.057307] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.057312] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.057315] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.057321] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=215, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.057327] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=1918563, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=215, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.058856] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.058867] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.060024] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164325C300CF-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301545, send_timestamp=1719305996758478) [2024-06-25 17:00:19.060313] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABF4533-00061661728FC8D9-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301546, send_timestamp=1719305996758767) [2024-06-25 17:00:19.062046] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B35-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1009, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=9, avg_log_batch_cnt=0, total_group_log_size=1098, avg_group_log_size=122) [2024-06-25 17:00:19.062911] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFF6-0-0] [lt=9] [PALF STAT LOG TASK TIME](palf_id=1009, self="10.191.69.51:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=91, avg_log_submit_to_slide_time=842) [2024-06-25 17:00:19.062916] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT HOT CACHE HIT RATE](palf_id=1009, read_size=1098, hit_cnt=9, read_cnt=9, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.066327] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.066335] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.066338] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.066340] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.066342] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.066574] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-000616432547EFF7-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301550, send_timestamp=1719305996765024) [2024-06-25 17:00:19.066872] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=9][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:53, local_retry_times:53, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.066878] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=6][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.066881] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.066883] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.066888] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.068297] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.068304] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=7][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.068306] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e68be20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019014313, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.068319] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=13][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e68be20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019014313, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.068329] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=10][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.068334] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.068338] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.068340] WDIAG [SQL] open (ob_result_set.cpp:157) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.068342] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.068345] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.068349] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=204, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.068352] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3] will sleep(sleep_us=100000, remain_us=3636629, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=204, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.075331] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.075344] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.075352] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:19.075359] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:19.075362] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.075366] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:19.075369] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.075371] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.075375] WDIAG [SQL.ENG] inner_get_next_batch_for_tsc (ob_table_scan_op.cpp:2015) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:19.075377] WDIAG [SQL.ENG] inner_get_next_batch (ob_table_scan_op.cpp:2000) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:19.075380] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] get next batch failed(ret=-4383, eval_ctx={batch_idx:0, batch_size:16, max_batch_size:16, frames_:0x2bb27d012e10}, id=2, op_name="PHY_TABLE_SCAN") [2024-06-25 17:00:19.075386] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1535) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4383] get next batch failed(ret=-4383) [2024-06-25 17:00:19.075388] WDIAG [SQL.ENG] get_next_row_vectorizely (ob_operator.cpp:1463) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] Failed to get_next_row vectorizedly(ret=-4383) [2024-06-25 17:00:19.075391] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_transmit_op.cpp:27) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] get next row from child failed(ret=-4383) [2024-06-25 17:00:19.075395] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4383] inner get next row failed(ret=-4383, type=14, op="PHY_DIRECT_TRANSMIT", op_id=1) [2024-06-25 17:00:19.075398] WDIAG [SQL.EXE] sync_send_result (ob_remote_executor_processor.cpp:283) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:19.075402] WDIAG [SQL.EXE] execute_remote_plan (ob_remote_executor_processor.cpp:469) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4383] sync send result failed(ret=-4383) [2024-06-25 17:00:19.075406] WDIAG [SQL.EXE] process (ob_remote_executor_processor.cpp:969) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] execute remote plan failed(ret=-4383, local_tenant_schema_version=1718209168713032, exec_ctx_.get_task_exec_ctx().get_query_tenant_begin_schema_version()=1718209168713032) [2024-06-25 17:00:19.077307] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62249][pnio1][T1001][YB420ABF4533-00061661718F6532-0-0] [lt=5][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.082951] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.082960] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.082964] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6790d360, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019028996, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.082981] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6790d360, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019028996, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.082992] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.082999] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.083002] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.083006] WDIAG [SQL] open (ob_result_set.cpp:157) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.083008] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.083011] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.083016] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=78, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.083022] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5] will sleep(sleep_us=78000, remain_us=22741957, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=78, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.085168] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.090439] INFO [COMMON] process (ob_rpc_processor_simple.cpp:2405) [95444][T1002_L0_G19][T1002][YB420ABDC060-000616431A273CD4-0-0] [lt=11] handle ObQueryLSIsValidMember requeset(arg_={tenant_id_:1002, self_addr_:"10.189.192.96:2882", ls_array_:[{id:1006}]}) [2024-06-25 17:00:19.090449] INFO [CLOG] get_member_gc_stat (ob_log_handler.cpp:1509) [95444][T1002_L0_G19][T1002][YB420ABDC060-000616431A273CD4-0-0] [lt=8] get_member_gc_stat(is_valid_member=true, stat=1, member_list=5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, learner_list={learner_num:0, learner_array:[]}, addr="10.189.192.96:2882", this={role:1, proposal_id:1, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1006}) [2024-06-25 17:00:19.092544] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.104195] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC07C-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1, self="10.191.69.51:2882", committed size=4306) [2024-06-25 17:00:19.105015] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A081-0-0] [lt=10] [PALF STAT LOG TASK TIME](palf_id=1, self="10.191.69.51:2882", total_slide_log_cnt=12, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=93, avg_log_submit_to_slide_time=829) [2024-06-25 17:00:19.105023] INFO [PALF] read (log_cache.cpp:100) [62747][T1_ReplaySrv4][T1][Y0-0000000000000000-0-0] [lt=3] [PALF STAT HOT CACHE HIT RATE](palf_id=1, read_size=3889, hit_cnt=12, read_cnt=12, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.108666] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=9] new mysql sessid created(conn.sessid_=3221510720, support_ssl=false) [2024-06-25 17:00:19.108684] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=8] sm conn init succ(conn.sessid_=3221510720, sess.client_addr_="10.191.69.51:47998") [2024-06-25 17:00:19.108691] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=5] accept one succ(*s={this:0x2b3ac6e24d70, session_id:3221510720, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:907}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:19.108884] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x2b3ac6e24d70, session_id:3221510720, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:907}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:19.108893] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=8] kill and revert session(conn.sessid_=3221510720, proxy_sessid=0, server_id=1, ret=0) [2024-06-25 17:00:19.108898] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:918) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=5] sock ref clean, do destroy(*s={this:0x2b3ac6e24d70, session_id:3221510720, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:907}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:19.108905] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:223) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=6] mark session id unused(conn.sessid_=3221510720) [2024-06-25 17:00:19.108908] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [62489][sql_nio10][T0][Y0-0000000000000000-0-0] [lt=2] connection close(sessid=3221510720, proxy_sessid=0, tenant_id=0, server_id=1, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=false, ret=0, trace_id=Y0-0000000000000000-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:0], disconnect_state=0) [2024-06-25 17:00:19.109673] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=7] re_register_parent reason(palf_id=1006, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306015007500, last_first_register_time_us=1713332560377970, register_time_us=1719306015007500, curr_time_us=1719306019109671) [2024-06-25 17:00:19.109697] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=11] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1006, self="10.191.69.51:2882", reason=1, leader="10.191.69.56:2882", register_time_us=1719306019109685, last_submit_register_req_time_us=1719306019109685, register_parent_reason=1) [2024-06-25 17:00:19.109706] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=8] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1006, self="10.191.69.51:2882") [2024-06-25 17:00:19.109822] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=9][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:163, local_retry_times:163, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.109836] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=11][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.109840] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.109843] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.109851] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.111022] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=9][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:191, local_retry_times:191, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.111033] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.111037] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.111040] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.111046] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.120716] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.120728] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.120732] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6d6bf4a0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019066854, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.120750] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=18][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6d6bf4a0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019066854, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.120761] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.120768] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.120772] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.120774] WDIAG [SQL] open (ob_result_set.cpp:157) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.120777] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.120780] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.120784] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=54, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.120790] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5] will sleep(sleep_us=54000, remain_us=25606386, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=54, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.124105] WDIAG [STORAGE.TRANS] process_ (ob_gti_rpc.h:178) [62242][pnio1][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=8][errcode=0] gti rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:19.125184] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBADE-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:19.125410] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC89D-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:19.126255] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFF8-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=91, avg_log_submit_to_slide_time=837) [2024-06-25 17:00:19.129763] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.129786] INFO eloop_run (eloop.c:143) [62232][pnio1][T0][Y0-0000000000000000-0-0] [lt=13] PNIO [ratelimit] time: 1719306019129786, bytes: 29137865946318, bw: 0.739710 MB/s, add_ts: 1000191, add_bytes: 775790 [2024-06-25 17:00:19.129799] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.129802] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.129806] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.129810] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.129998] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325962ABD-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301540, send_timestamp=1719305996828457) [2024-06-25 17:00:19.130038] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-0006164325962ABE-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301556, send_timestamp=1719305996828482) [2024-06-25 17:00:19.131221] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=10] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=1) [2024-06-25 17:00:19.132550] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.132635] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301554, send_timestamp=1719305996831080) [2024-06-25 17:00:19.133399] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF59C7-0-0] [lt=8] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b61c69470, extra_info="") [2024-06-25 17:00:19.133444] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.133475] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=64, this=0x2b3b61c733b0, extra_info="palf_id:1006") [2024-06-25 17:00:19.133504] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3b61c735e0, extra_info="palf_id:1006") [2024-06-25 17:00:19.133690] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ABF-0-0] [lt=6] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b61c696a0, extra_info="palf_id:1006") [2024-06-25 17:00:19.133694] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ABF-0-0] [lt=4] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=280, this=0x2b3b61c698d0, extra_info="palf_id:1006") [2024-06-25 17:00:19.137448] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.138089] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.138493] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABF4538-000616694DDF523B-0-0] [lt=9] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1006}}) [2024-06-25 17:00:19.138508] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABF4538-000616694DDF523B-0-0] [lt=7] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306018870127002, v:0}, arg={tenant_id:1004, ls_id:{id:1006}}, result={tenant_id:1004, ls_id:{id:1006}, cur_readable_scn:{val:1719306018870127002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:19.140924] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] [2024-06-25 17:00:19.140931] WDIAG [SQL.EXE] setup_next_scanner (ob_direct_receive_op.cpp:207) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4012] error occurring in the remote sql execution, please use the current TRACE_ID to grep the original error message on the remote_addr.(ret=-4012, remote_addr="10.186.69.171:2882") [2024-06-25 17:00:19.140936] WDIAG [SQL.EXE] inner_open (ob_direct_receive_op.cpp:53) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] failed to setup first scanner(ret=-4012) [2024-06-25 17:00:19.140940] WDIAG [SQL.ENG] open (ob_operator.cpp:714) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] Open this operator failed(ret=-4012, op_type="PHY_DIRECT_RECEIVE") [2024-06-25 17:00:19.140946] WDIAG [SQL.EXE] open (ob_execute_result.cpp:104) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] open operator failed(ret=-4012) [2024-06-25 17:00:19.140949] WDIAG [SQL] open_result (ob_result_set.cpp:210) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail open main query(ret=-4012) [2024-06-25 17:00:19.140954] WDIAG [SQL] open (ob_result_set.cpp:159) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.140956] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.140959] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.140965] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] execute failed(ret=-4012, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:19.140971] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:19.140978] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.140981] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.140984] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:19.140990] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:19.140996] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4012, ret=-4012) [2024-06-25 17:00:19.140999] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.141004] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1) [2024-06-25 17:00:19.141008] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:19.141013] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] query failed(ret=-4012, conn=0x2b3a7c6db470, start=1719306019138350, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:19.141018] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.141040] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] get all ls info by persistent_ls_ failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1001) [2024-06-25 17:00:19.141046] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1247) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] the sql used to get ls locations error, treat as timeout(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", previous_ret=-4012, ls_ids=[{id:1}]) [2024-06-25 17:00:19.141054] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4664] batch renew ls locations failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.141059] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] renew location failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.141063] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get location(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.141067] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get log stream location leader(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.141074] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4664] get leader failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:3000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.141078] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.141084] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4664] fetch new range failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT") [2024-06-25 17:00:19.141087] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:19.141093] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:19.141096] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.141101] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:19.141104] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.141106] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:19.141112] WDIAG [SQL.ENG] inner_get_next_row_for_tsc (ob_table_scan_op.cpp:1913) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:19.141115] WDIAG [SQL.ENG] inner_get_next_row_implement (ob_table_scan_op.cpp:1900) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:19.141118] WDIAG [SQL.ENG] inner_get_next_row (ob_table_scan_op.cpp:2861) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] get next row failed(ret=-4383) [2024-06-25 17:00:19.141123] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4383] inner get next row failed(ret=-4383, type=3, op="PHY_TABLE_SCAN", op_id=0) [2024-06-25 17:00:19.141127] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4383] get next row from operator failed(ret=-4383) [2024-06-25 17:00:19.141131] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] get next row from exec result failed(ret=-4383) [2024-06-25 17:00:19.141135] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4383] get_next_row failed(ret=-4383, has_tenant_resource()=true) [2024-06-25 17:00:19.141138] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')"}) [2024-06-25 17:00:19.141142] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1002, executor={ObIExecutor:, sql:"SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')"}, retry_cnt=34, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1718209168713032) [2024-06-25 17:00:19.141146] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] will sleep(sleep_us=34000, remain_us=58992, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=34, v.err_=-4383, timeout_timestamp=1719306019200138) [2024-06-25 17:00:19.141862] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62415][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142153] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62429][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142245] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62426][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142288] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62430][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142414] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62416][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142632] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62420][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142634] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62432][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142653] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62425][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142698] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62417][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142713] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62421][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142760] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62423][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142869] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62422][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.142984] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62431][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.143007] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62419][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.143017] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62418][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.143087] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62424][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.143140] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62427][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.143470] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62428][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.145353] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.149115] INFO [STORAGE.BLKMGR] inspect_bad_block (ob_block_manager.cpp:1712) [62325][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=13] check macro block(block_info={ref_cnt:1, is_free:false, access_time:1717170479939398}, time_interval=2135539209716) [2024-06-25 17:00:19.149839] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF78EC-0-0] [lt=5] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86d1d470, extra_info="") [2024-06-25 17:00:19.149850] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF78EC-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1333, avg_group_log_size=121) [2024-06-25 17:00:19.149931] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=81, this=0x2b3a86d273b0, extra_info="palf_id:1007") [2024-06-25 17:00:19.149951] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a86d275e0, extra_info="palf_id:1007") [2024-06-25 17:00:19.149997] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:19.150134] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFF9-0-0] [lt=4] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a86d1d6a0, extra_info="palf_id:1007") [2024-06-25 17:00:19.150140] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFF9-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=287, this=0x2b3a86d1d8d0, extra_info="palf_id:1007") [2024-06-25 17:00:19.152463] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:211, local_retry_times:211, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.152473] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.152476] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.152479] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.152485] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.154744] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABE7C-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.154830] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:132, local_retry_times:132, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.154841] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.154846] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.154848] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.154854] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.155179] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.157391] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=7][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:215, local_retry_times:215, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.157399] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=7][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.157402] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.157405] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.157411] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.158465] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.158477] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.160076] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325C300D3-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301548, send_timestamp=1719305996858527) [2024-06-25 17:00:19.161082] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:78, local_retry_times:78, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.161092] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.161095] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.161098] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.161104] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.163486] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=7][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.163495] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.163498] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e775ba0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019108899, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.163520] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=22][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e775ba0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019108899, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.163532] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.163539] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.163543] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.163546] WDIAG [SQL] open (ob_result_set.cpp:157) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.163549] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.163553] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.163557] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=164, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.163563] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=9785704, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=164, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.164655] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.164665] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.164669] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6ab5c780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019111013, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.164689] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=20][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6ab5c780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019111013, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.164701] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999998, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.164707] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.164712] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.164714] WDIAG [SQL] open (ob_result_set.cpp:157) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.164718] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.164721] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}) [2024-06-25 17:00:19.164726] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, retry_cnt=192, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.164732] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=5480609, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=192, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.166386] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.166392] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.166395] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.166397] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.166400] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.166475] INFO [LIB] stat (utility.h:1214) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3a8ce1cc70, extra_info="") [2024-06-25 17:00:19.167067] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [63199][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=9] ====== tenant freeze timer task ====== [2024-06-25 17:00:19.168410] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:204, local_retry_times:204, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.168418] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=7][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.168420] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.168423] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.168429] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.172291] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=0][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:19.172299] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:19.172876] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.173982] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63309][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=7] start do ls ha handler(ls_id_array_=[{id:1010}, {id:1008}, {id:1009}, {id:1006}, {id:1007}]) [2024-06-25 17:00:19.174339] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289111621632, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:19.174851] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:54, local_retry_times:54, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.174862] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.174865] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.174868] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.174875] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.175206] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:34, local_retry_times:34, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.175215] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.175218] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.175220] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.175630] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.177325] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62238][pnio1][T1001][YB420ABF4533-00061661718F6533-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.178777] INFO [STORAGE] runTimerTask (ob_locality_manager.cpp:708) [62328][LocalityReload][T0][Y0-0000000000000000-0-0] [lt=7] runTimer to refresh locality_info(ret=0) [2024-06-25 17:00:19.179204] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-4012] [2024-06-25 17:00:19.179210] WDIAG [SQL.EXE] setup_next_scanner (ob_direct_receive_op.cpp:207) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] error occurring in the remote sql execution, please use the current TRACE_ID to grep the original error message on the remote_addr.(ret=-4012, remote_addr="10.186.69.171:2882") [2024-06-25 17:00:19.179214] WDIAG [SQL.EXE] inner_open (ob_direct_receive_op.cpp:53) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] failed to setup first scanner(ret=-4012) [2024-06-25 17:00:19.179218] WDIAG [SQL.ENG] open (ob_operator.cpp:714) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] Open this operator failed(ret=-4012, op_type="PHY_DIRECT_RECEIVE") [2024-06-25 17:00:19.179222] WDIAG [SQL.EXE] open (ob_execute_result.cpp:104) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] open operator failed(ret=-4012) [2024-06-25 17:00:19.179225] WDIAG [SQL] open_result (ob_result_set.cpp:210) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] fail open main query(ret=-4012) [2024-06-25 17:00:19.179227] WDIAG [SQL] open (ob_result_set.cpp:159) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.179235] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.179239] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.179242] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute failed(ret=-4012, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:19.179248] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:19.179254] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.179258] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.179260] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:19.179264] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:19.179270] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4012, ret=-4012) [2024-06-25 17:00:19.179273] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.179277] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1) [2024-06-25 17:00:19.179281] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:19.179284] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] query failed(ret=-4012, conn=0x2b3ac92dd750, start=1719306019176694, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:19.179290] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.179310] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] get all ls info by persistent_ls_ failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1001) [2024-06-25 17:00:19.179315] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1247) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4664] the sql used to get ls locations error, treat as timeout(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", previous_ret=-4012, ls_ids=[{id:1}]) [2024-06-25 17:00:19.179319] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4664] batch renew ls locations failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.179324] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] renew location failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.179328] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get location(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.179331] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get log stream location leader(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.179337] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4664] get leader failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.179341] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.179346] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] fetch new range failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT") [2024-06-25 17:00:19.180321] INFO [SHARE] set_server_locality_array (ob_server_locality_cache.cpp:346) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=8] set_server_locality_array success(server_locality_array=[{inited:true, addr:"10.186.69.171:2882", zone:"zone3", zone_type:0, idc:"wg_sh16", region:"sys_region", is_active:true, start_service_time:1713323605265064, server_stop_time:0, server_status:1}, {inited:true, addr:"10.186.69.173:2882", zone:"zone3", zone_type:0, idc:"wg_sh16", region:"sys_region", is_active:true, start_service_time:1713323615484318, server_stop_time:0, server_status:1}, {inited:true, addr:"10.186.69.176:2882", zone:"zone4", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323605225767, server_stop_time:0, server_status:1}, {inited:true, addr:"10.186.69.177:2882", zone:"zone4", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323615512327, server_stop_time:0, server_status:1}, {inited:true, addr:"10.189.192.96:2882", zone:"zone5", zone_type:0, idc:"wg_88", region:"sys_region", is_active:true, start_service_time:1713323605377346, server_stop_time:0, server_status:1}, {inited:true, addr:"10.189.192.97:2882", zone:"zone5", zone_type:0, idc:"wg_88", region:"sys_region", is_active:true, start_service_time:1713323615580956, server_stop_time:0, server_status:1}, {inited:true, addr:"10.191.69.51:2882", zone:"zone1", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323604978888, server_stop_time:0, server_status:1}, {inited:true, addr:"10.191.69.52:2882", zone:"zone1", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323615397450, server_stop_time:0, server_status:1}, {inited:true, addr:"10.191.69.56:2882", zone:"zone2", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323605233519, server_stop_time:0, server_status:1}, {inited:true, addr:"10.191.69.57:2882", zone:"zone2", zone_type:0, idc:"wg_sh8", region:"sys_region", is_active:true, start_service_time:1713323615481577, server_stop_time:0, server_status:1}], has_readonly_zone=false) [2024-06-25 17:00:19.180353] INFO [SHARE] load_region (ob_locality_table_operator.cpp:336) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=27] tenant_schema primary_zone is NULL, or no need calc region priority(tenant_id=1, primary_zone=RANDOM) [2024-06-25 17:00:19.180360] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=6] add_locality_zone success(locality_zone={tenant_id:1, region_priority:18446744073709551615}) [2024-06-25 17:00:19.180366] INFO [SHARE] load_region (ob_locality_table_operator.cpp:336) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=4] tenant_schema primary_zone is NULL, or no need calc region priority(tenant_id=1001, primary_zone=RANDOM) [2024-06-25 17:00:19.180370] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=3] add_locality_zone success(locality_zone={tenant_id:1001, region_priority:18446744073709551615}) [2024-06-25 17:00:19.180375] INFO [SHARE] load_region (ob_locality_table_operator.cpp:336) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=2] tenant_schema primary_zone is NULL, or no need calc region priority(tenant_id=1002, primary_zone=RANDOM) [2024-06-25 17:00:19.180379] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=3] add_locality_zone success(locality_zone={tenant_id:1002, region_priority:18446744073709551615}) [2024-06-25 17:00:19.180385] INFO [SHARE] load_region (ob_locality_table_operator.cpp:346) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=2] get_primary_zone_prioriry(primary_zone=zone1,zone2,zone3,zone4;zone5, tenant_primary_region_array=[{region:"sys_region", region_priority:0, zone_array:["zone1", "zone2", "zone3", "zone4", "zone5"]}], locality_info={version:0, local_region:"sys_region", local_zone:"zone1", local_idc:"wg_sh8", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone3", "zone4", "zone5", "zone1", "zone2"]}], locality_zone_array:[{tenant_id:1, region_priority:18446744073709551615}, {tenant_id:1001, region_priority:18446744073709551615}, {tenant_id:1002, region_priority:18446744073709551615}]}) [2024-06-25 17:00:19.180399] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=14] add_locality_zone success(locality_zone={tenant_id:1003, region_priority:0}) [2024-06-25 17:00:19.180403] INFO [SHARE] load_region (ob_locality_table_operator.cpp:346) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=2] get_primary_zone_prioriry(primary_zone=zone1,zone2,zone3,zone4;zone5, tenant_primary_region_array=[{region:"sys_region", region_priority:0, zone_array:["zone1", "zone2", "zone3", "zone4", "zone5"]}], locality_info={version:0, local_region:"sys_region", local_zone:"zone1", local_idc:"wg_sh8", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone3", "zone4", "zone5", "zone1", "zone2"]}], locality_zone_array:[{tenant_id:1, region_priority:18446744073709551615}, {tenant_id:1001, region_priority:18446744073709551615}, {tenant_id:1002, region_priority:18446744073709551615}, {tenant_id:1003, region_priority:0}]}) [2024-06-25 17:00:19.180411] INFO [SHARE] load_region (ob_locality_table_operator.cpp:363) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=8] add_locality_zone success(locality_zone={tenant_id:1004, region_priority:0}) [2024-06-25 17:00:19.180415] INFO [SHARE] load_region (ob_locality_table_operator.cpp:373) [62327][LocltyRefTask][T0][Y0-0000000000000000-0-0] [lt=3] load region(ret=0, locality_info={version:0, local_region:"sys_region", local_zone:"zone1", local_idc:"wg_sh8", local_zone_type:0, local_zone_status:2, locality_region_array:[{region:"sys_region", region_priority:18446744073709551615, zone_array:["zone3", "zone4", "zone5", "zone1", "zone2"]}], locality_zone_array:[{tenant_id:1, region_priority:18446744073709551615}, {tenant_id:1001, region_priority:18446744073709551615}, {tenant_id:1002, region_priority:18446744073709551615}, {tenant_id:1003, region_priority:0}, {tenant_id:1004, region_priority:0}]}) [2024-06-25 17:00:19.181376] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.183586] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] [2024-06-25 17:00:19.183592] WDIAG [SQL.EXE] setup_next_scanner (ob_direct_receive_op.cpp:207) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] error occurring in the remote sql execution, please use the current TRACE_ID to grep the original error message on the remote_addr.(ret=-4012, remote_addr="10.186.69.171:2882") [2024-06-25 17:00:19.183596] WDIAG [SQL.EXE] inner_open (ob_direct_receive_op.cpp:53) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] failed to setup first scanner(ret=-4012) [2024-06-25 17:00:19.183599] WDIAG [SQL.ENG] open (ob_operator.cpp:714) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] Open this operator failed(ret=-4012, op_type="PHY_DIRECT_RECEIVE") [2024-06-25 17:00:19.183601] WDIAG [SQL.EXE] open (ob_execute_result.cpp:104) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] open operator failed(ret=-4012) [2024-06-25 17:00:19.183604] WDIAG [SQL] open_result (ob_result_set.cpp:210) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail open main query(ret=-4012) [2024-06-25 17:00:19.183607] WDIAG [SQL] open (ob_result_set.cpp:159) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.183610] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.183613] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.183618] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] execute failed(ret=-4012, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:19.183622] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:19.183628] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.183631] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.183633] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:19.183639] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:19.183644] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4012, ret=-4012) [2024-06-25 17:00:19.183648] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.183652] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1) [2024-06-25 17:00:19.183655] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:19.183658] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] query failed(ret=-4012, conn=0x2b3a78d44050, start=1719306019181411, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:19.183664] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.183679] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] get all ls info by persistent_ls_ failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1001) [2024-06-25 17:00:19.183684] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1247) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4664] the sql used to get ls locations error, treat as timeout(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", previous_ret=-4012, ls_ids=[{id:1}]) [2024-06-25 17:00:19.183687] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] batch renew ls locations failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.183692] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] renew location failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.183695] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get location(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.183699] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get log stream location leader(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.183705] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] get leader failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.183708] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3] fetch new DAS ID range finish(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.183714] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] fetch new range failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT") [2024-06-25 17:00:19.185224] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.189463] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.196314] WDIAG [RPC] send (ob_poc_rpc_proxy.h:156) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] sync rpc execute fail(ret=-4012, addr="10.186.69.171:2882", pcode=1308) [2024-06-25 17:00:19.196325] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=9][errcode=-4012] rpc task_execute fail(ret=-4012, tenant_id=1, svr="10.186.69.171:2882", timeout=4058, timeout_timestamp=1719306019200138) [2024-06-25 17:00:19.196331] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] task execute failed(ret=-4012) [2024-06-25 17:00:19.196335] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute with sql failed(ret=-4012) [2024-06-25 17:00:19.196338] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail execute plan(ret=-4012) [2024-06-25 17:00:19.196342] WDIAG [SQL] open (ob_result_set.cpp:157) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] execute plan failed(ret=-4012) [2024-06-25 17:00:19.196344] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:19.196347] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.196352] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] execute failed(ret=-4012, tenant_id=1, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:19.196357] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:19.196362] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.196366] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:19.196368] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:19.196373] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:19.196378] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4012, ret=-4012) [2024-06-25 17:00:19.196381] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.196385] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1) [2024-06-25 17:00:19.196389] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:19.196392] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:382) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] query failed(ret=-4012, conn=0x2b3a78d45a30, start=1719306019195828, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port, cluster_id=1713323324) [2024-06-25 17:00:19.196396] WDIAG [SHARE.PT] batch_get (ob_persistent_ls_table.cpp:864) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", exec_tenant_id=1, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1001 AND ls_id IN (1) ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.196412] WDIAG [SHARE.PT] batch_get (ob_ls_table_operator.cpp:399) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] get all ls info by persistent_ls_ failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1001) [2024-06-25 17:00:19.196417] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1247) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] the sql used to get ls locations error, treat as timeout(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", previous_ret=-4012, ls_ids=[{id:1}]) [2024-06-25 17:00:19.196421] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] batch renew ls locations failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.196424] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] renew location failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.196429] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] fail to get location(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.196432] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4664] fail to get log stream location leader(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.196438] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4664] get leader failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.196442] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.196447] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4664] fetch new range failed(ret=-4664, ret="OB_GET_LOCATION_TIME_OUT") [2024-06-25 17:00:19.198301] INFO pktc_resp_cb_on_msg (pktc_resp.h:70) [62247][pnio1][T0][Y0-0000000000000000-0-0] [lt=7] PNIO resp cb not found: packet_id=1035876821 [2024-06-25 17:00:19.199313] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.199543] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=22] get_max_applied_scn(scn={val:1713323697228952000, v:0}, this={ls_id_:{id:1008}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713323697228952000, v:0}}) [2024-06-25 17:00:19.199557] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=12] get_min_unreplayed_log_info(lsn={lsn:74143917315}, scn={val:1719306019044617001, v:0}, this={ls_id_:{id:1008}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306019126252, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:74143917315}, next_to_submit_scn_:{val:1719306019044617001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713323697228952000, v:0}, iterator_:{iterator_impl:{buf_:0x2b3bda663000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:74143917193}, end_lsn:{lsn:74143917315}, read_buf:{buf_len_:73728, buf_:0x2b3bda663000}, block_size:67104768, log_storage_:0x2b3a86ded470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306019044617000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306019044617000, v:0}, data_checksum:3647337712, flag:0}}, init_mode_version:0, accumulate_checksum:2515280368, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306019044617000, v:0}}}}}) [2024-06-25 17:00:19.201253] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=5] LogLoopThread round_cost_time(us)(round_cost_time=13) [2024-06-25 17:00:19.206264] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.206273] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.206277] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78dd8400, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019152034, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.206294] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78dd8400, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019152034, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.206306] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.206313] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=6][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.206316] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.206319] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.206322] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.206326] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.206330] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=212, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.206336] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=2402459, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=212, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.206713] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.208634] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.208648] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=14526175, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=133, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.211087] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.211102] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1764788, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=216, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.212506] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1719306019200138, ctx_timeout_ts=-1, worker_timeout_ts=1719306019200138, default_timeout=1000000) [2024-06-25 17:00:19.212514] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.212517] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] batch renew ls locations failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.212521] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] renew location failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.212526] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] fail to get location(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.212529] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail to get log stream location leader(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.212534] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] get leader failed(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.212537] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2] fetch new DAS ID range finish(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.212541] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] fetch new range failed(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.214843] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.214857] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=79000, remain_us=22610122, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=79, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.215170] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=5] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647473, offset=60295969}, end_cursor:ObLogCursor{file_id=100, log_id=27647474, offset=60296210}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb80050, len:1247, log_data_len:241, seq:27647473, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:19.216290] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8012-0-0] [lt=9] push gts task success(*task={:""}) [2024-06-25 17:00:19.217124] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.218187] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8012-0-0] [lt=9] rpc statistics(total_trans_msg_count=2, total_batch_msg_count=0) [2024-06-25 17:00:19.218786] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=14] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647474, offset=60296210}, end_cursor:ObLogCursor{file_id=100, log_id=27647475, offset=60296459}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb90050, len:1006, log_data_len:249, seq:27647474, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:19.219814] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=4] ====== check clog disk timer task ====== [2024-06-25 17:00:19.219823] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=6] get_disk_usage(ret=0, capacity(MB):=32384, used(MB):=25883) [2024-06-25 17:00:19.219834] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=5] cannot_recycle_log_size statistics(cannot_recycle_log_size=229969377, threshold=10187125555, need_update_checkpoint_scn=false) [2024-06-25 17:00:19.221963] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3483019, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=205, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.223487] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647475, offset=60296459}, end_cursor:ObLogCursor{file_id=100, log_id=27647476, offset=60296708}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb9c050, len:757, log_data_len:249, seq:27647475, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:19.223601] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=5] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647476, offset=60296708}, end_cursor:ObLogCursor{file_id=100, log_id=27647477, offset=60296957}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fba0050, len:508, log_data_len:249, seq:27647476, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:19.223623] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=22] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:19.223745] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=5] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647477, offset=60296957}, end_cursor:ObLogCursor{file_id=100, log_id=27647478, offset=60297206}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fb94050, len:4355, log_data_len:249, seq:27647477, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:19.225461] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC8A1-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86de7470, extra_info="") [2024-06-25 17:00:19.225550] INFO [LIB] stat (utility.h:1197) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=71, this=0x2b3a86df13b0, extra_info="palf_id:1008") [2024-06-25 17:00:19.225562] INFO [LIB] stat (utility.h:1214) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a8ce1daf0, extra_info="") [2024-06-25 17:00:19.225576] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a86df15e0, extra_info="palf_id:1008") [2024-06-25 17:00:19.226322] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFFC-0-0] [lt=6] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a86de76a0, extra_info="palf_id:1008") [2024-06-25 17:00:19.226327] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547EFFC-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=840, this=0x2b3a86de78d0, extra_info="palf_id:1008") [2024-06-25 17:00:19.228618] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.228635] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=55000, remain_us=25498541, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=55, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.229814] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.229847] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.229852] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.229854] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.229857] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.229890] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1333) [2024-06-25 17:00:19.230059] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164325962AC2-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305996928512) [2024-06-25 17:00:19.230082] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164325962AC3-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301540, send_timestamp=1719305996928542) [2024-06-25 17:00:19.232016] INFO eloop_run (eloop.c:143) [62250][pnio2][T0][Y0-0000000000000000-0-0] [lt=9] PNIO [ratelimit] time: 1719306019232016, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007622, add_bytes: 0 [2024-06-25 17:00:19.232575] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.233653] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AC4-0-0] [lt=6] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=79, avg_log_submit_to_slide_time=275) [2024-06-25 17:00:19.233662] INFO [PALF] read (log_cache.cpp:100) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=14] [PALF STAT HOT CACHE HIT RATE](palf_id=1006, read_size=1333, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.238144] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.244561] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1719306019200138, ctx_timeout_ts=-1, worker_timeout_ts=1719306019200138, default_timeout=1000000) [2024-06-25 17:00:19.244569] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=8][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.244573] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] batch renew ls locations failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.244580] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] renew location failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.244583] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail to get location(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.244587] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] fail to get log stream location leader(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.244594] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] get leader failed(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.244599] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.244604] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] fetch new range failed(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.244827] INFO [CLOG.EXTLOG] resize_log_ext_handler_ (ob_cdc_service.cpp:632) [62693][T1_CdcSrv][T1][Y0-0000000000000000-0-0] [lt=10] finish to resize log external storage handler(current_ts=1719306019244826, tenant_max_cpu=6, valid_ls_count=0, other_ls_count=0, new_concurrency=0) [2024-06-25 17:00:19.244906] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [99399][T1004_LSArchive][T1004][YB420ABF4533-00061642492E6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:19.250974] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.260125] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-0006164325C300D7-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301547, send_timestamp=1719305996958577) [2024-06-25 17:00:19.260408] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABF4538-00061669548FCBFC-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301542, send_timestamp=1719305996958866) [2024-06-25 17:00:19.264792] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.266440] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.266447] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.266450] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.266452] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.266455] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.266673] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-000616432547EFFE-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305996965128) [2024-06-25 17:00:19.272542] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.277268] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [63377][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=10] ====== tenant freeze timer task ====== [2024-06-25 17:00:19.277282] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [63377][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=9][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-06-25 17:00:19.277381] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62246][pnio1][T1001][YB420ABF4533-00061661718F6534-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.278548] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.284269] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061644F985525A-0-0] [lt=7] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1005, child={server:"10.186.69.176:2882", region:"sys_region", register_time_us:1719306019284181, keepalive_ts:-1}) [2024-06-25 17:00:19.284278] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061644F985525A-0-0] [lt=9] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1005, self="10.191.69.51:2882", child={server:"10.186.69.176:2882", region:"sys_region", register_time_us:1719306019284181, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:19.285283] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=13][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.288352] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.306396] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:212, local_retry_times:212, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.306407] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.306411] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.306413] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.306419] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.308666] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1719306019200138, ctx_timeout_ts=-1, worker_timeout_ts=1719306019200138, default_timeout=1000000) [2024-06-25 17:00:19.308673] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.308676] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] batch renew ls locations failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.308682] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] renew location failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.308685] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail to get location(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.308689] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail to get log stream location leader(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.308694] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] get leader failed(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.308698] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3] fetch new DAS ID range finish(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.308703] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] fetch new range failed(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.313370] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [62671][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=10] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-06-25 17:00:19.316726] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.317218] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=9632049, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=165, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.318531] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5326809, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=193, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.322028] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=10][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.325273] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBAE6-0-0] [lt=5] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b74f470, extra_info="") [2024-06-25 17:00:19.325365] INFO [LIB] stat (utility.h:1197) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=79, this=0x2b3b5b7593b0, extra_info="palf_id:1007") [2024-06-25 17:00:19.325374] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1211) [2024-06-25 17:00:19.325383] INFO [LIB] stat (utility.h:1214) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3afcbffaf0, extra_info="") [2024-06-25 17:00:19.325398] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x2b3b5b7595e0, extra_info="palf_id:1007") [2024-06-25 17:00:19.326163] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AC6-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b5b74f6a0, extra_info="palf_id:1007") [2024-06-25 17:00:19.326170] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AC6-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=848, this=0x2b3b5b74f8d0, extra_info="palf_id:1007") [2024-06-25 17:00:19.326174] INFO [CLOG] handle_submit_task_ (ob_log_replay_service.cpp:1208) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=9] succ to submit log task to replay service(to_submit_lsn={lsn:5896353983088}, to_submit_scn={val:1719306019044617002, v:0}, replay_status={ls_id_:{id:1007}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306019326161, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:237788, ret_code_:0}}, next_to_submit_lsn_:{lsn:5896353983154}, next_to_submit_scn_:{val:1719306019044617003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713332543410073001, v:0}, iterator_:{iterator_impl:{buf_:0x2b4f5c205000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:5896353983032}, end_lsn:{lsn:5896353983154}, read_buf:{buf_len_:3682304, buf_:0x2b4f5c205000}, block_size:67104768, log_storage_:0x2b3b5b755470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306019044617002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306019044617002, v:0}, data_checksum:2442502576, flag:0}}, init_mode_version:0, accumulate_checksum:287639555, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306019044617002, v:0}}}}}) [2024-06-25 17:00:19.327756] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABF4533-0006166171EF4868-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301551, send_timestamp=1719305997026205) [2024-06-25 17:00:19.329656] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [62719][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=20] get_max_applied_scn(scn={val:1, v:0}, this={ls_id_:{id:1}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1, v:0}}) [2024-06-25 17:00:19.329868] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.329898] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.329902] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.329905] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.329907] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.330006] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1005, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.330417] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95434][T1004_L0_G1][T1004][YB420ABA45AD-0006164650E9E66C-0-0] [lt=3] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1039, child={server:"10.186.69.173:2882", region:"sys_region", register_time_us:1719306019329304, keepalive_ts:-1}) [2024-06-25 17:00:19.330426] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95434][T1004_L0_G1][T1004][YB420ABA45AD-0006164650E9E66C-0-0] [lt=9] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1039, self="10.191.69.51:2882", child={server:"10.186.69.173:2882", region:"sys_region", register_time_us:1719306019329304, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:19.331295] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=15] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=0) [2024-06-25 17:00:19.332631] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.337489] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=56000, remain_us=25389687, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=56, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.338198] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=10][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.340415] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95434][T1004_L0_G1][T1004][YB420ABF4538-000616694DAF5331-0-0] [lt=6] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1005, child={server:"10.191.69.56:2882", region:"sys_region", register_time_us:1719306019338542, keepalive_ts:-1}) [2024-06-25 17:00:19.340421] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95434][T1004_L0_G1][T1004][YB420ABF4538-000616694DAF5331-0-0] [lt=6] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1005, self="10.191.69.51:2882", child={server:"10.191.69.56:2882", region:"sys_region", register_time_us:1719306019338542, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:19.343260] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [100423][T1003_FreInfoRe][T1003][Y0-0000000000000000-0-0] [lt=9] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-06-25 17:00:19.343539] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63467][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7156, mem_used=12478464) [2024-06-25 17:00:19.347557] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABF4533-0006166172BEBE90-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301548, send_timestamp=1719305997046008) [2024-06-25 17:00:19.347652] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=80000, remain_us=22477327, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=80, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.350034] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=7] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:19.353912] INFO [STORAGE.TRANS] print_statistics (ob_xa_define.cpp:467) [100461][T1003_ObXAHbWor][T1003][Y0-0000000000000000-0-0] [lt=11] xa statistics(total_active_xa_ctx_count=0, total_standby_clearup_count=0, total_success_xa_start=0, total_failure_xa_start=0, total_success_xa_prepare=0, total_failure_xa_prepare=0, total_success_xa_1pc_commit=0, total_failure_xa_1pc_commit=0, total_success_xa_2pc_commit=0, total_failure_xa_2pc_commit=0, total_failure_xa_rollback=0, total_success_dblink_promotion=0, total_failure_dblink_promotion=0, total_success_dblink=0, total_failure_dblink=0) [2024-06-25 17:00:19.360168] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-0006164325C300DB-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301544, send_timestamp=1719305997058624) [2024-06-25 17:00:19.360209] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.360219] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:19.360222] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78df9940, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019305674, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.360242] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=20][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78df9940, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306019305674, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:19.360255] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:19.360260] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:19.360264] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:19.360266] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:19.360269] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:19.360272] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:19.360276] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=213, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.360282] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=2248513, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=213, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.360462] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABF4538-00061669548FCBFF-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301542, send_timestamp=1719305997058919) [2024-06-25 17:00:19.362213] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B3D-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf09d470, extra_info="") [2024-06-25 17:00:19.362252] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62460][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.362318] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=78, this=0x2b3aaf0a73b0, extra_info="palf_id:1009") [2024-06-25 17:00:19.362338] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=6, this=0x2b3aaf0a75e0, extra_info="palf_id:1009") [2024-06-25 17:00:19.362462] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=14372361, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=134, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.363070] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62458][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.363080] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F001-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf09d6a0, extra_info="palf_id:1009") [2024-06-25 17:00:19.363086] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F001-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=845, this=0x2b3aaf09d8d0, extra_info="palf_id:1009") [2024-06-25 17:00:19.363110] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62455][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.363272] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62459][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.363457] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62456][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.363648] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62454][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.363712] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62457][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.364714] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [62725][T1_LSArchiveMgr][T1][YB420ABF4533-000616423A1E6725-0-0] [lt=9] gc stale ls task succ [2024-06-25 17:00:19.364724] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1611166, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=217, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.366504] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.366511] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.366513] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.366515] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.366518] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.366723] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC060-000616432547F002-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301541, send_timestamp=1719305997065181) [2024-06-25 17:00:19.367005] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62465][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367028] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62445][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367031] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62467][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367181] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62471][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367247] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62440][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367527] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62473][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367541] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62463][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367582] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62466][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367583] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62461][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367597] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62468][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=10] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367608] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62464][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367630] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62435][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367660] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62447][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367732] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62441][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367754] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62469][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367757] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62446][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367768] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62470][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367831] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62442][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367843] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62434][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367860] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62444][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.367864] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62443][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368123] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62452][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368125] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62462][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368184] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62449][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368184] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62448][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368184] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62437][BatchIO][T0][Y0-0000000000000000-0-0] [lt=9] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368192] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62451][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368227] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62438][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368228] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62436][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368245] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62439][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.368311] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62450][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.373727] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62475][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:19.374573] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=14] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289107427328, global_cache_size=552067284992, tenant_max_wash_size=83976305, tenant_min_wash_size=83976305, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:19.375727] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3329254, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=206, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.379449] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100271][T1004_L0_G0][T1004][YB420ABF4539-00061646327C8726-0-0] [lt=11][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.380445] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [62735][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[{id:1}]) [2024-06-25 17:00:19.383444] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100059][T1004_L0_G0][T1004][YB420ABF4539-00061646327C8726-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.385339] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.387744] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABF4533-0006166172BEBE95-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301555, send_timestamp=1719305997086188) [2024-06-25 17:00:19.393924] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.394927] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [62932][T1_L0_G0][T1][YB420ABA45AB-000616C80E7FB3EF-0-0] [lt=12] [RS_MGR] force set rs list(master_rs="10.186.69.171:2882") [2024-06-25 17:00:19.394939] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [62932][T1_L0_G0][T1][YB420ABA45AB-000616C80E7FB3EF-0-0] [lt=8] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"10.191.69.51:2882", server_id:1, rs_addr:"10.186.69.171:2882", rs_server_status:1, epoch_id:1}, hb_response={zone:"zone1", server:"10.191.69.51:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1713323604978888, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=13) [2024-06-25 17:00:19.396758] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.399061] INFO [STORAGE.TRANS] print_statistics (ob_xa_define.cpp:467) [99389][T1004_ObXAHbWor][T1004][Y0-0000000000000000-0-0] [lt=12] xa statistics(total_active_xa_ctx_count=0, total_standby_clearup_count=0, total_success_xa_start=0, total_failure_xa_start=0, total_success_xa_prepare=0, total_failure_xa_prepare=0, total_success_xa_1pc_commit=0, total_failure_xa_1pc_commit=0, total_success_xa_2pc_commit=0, total_failure_xa_2pc_commit=0, total_failure_xa_rollback=0, total_success_dblink_promotion=0, total_failure_dblink_promotion=0, total_success_dblink=0, total_failure_dblink=0) [2024-06-25 17:00:19.400512] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.402338] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.402358] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e68a080, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221528085, assoc_session_id:3221528085, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306019132035, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2bb3c49201f8, id:3221528085, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e68a080}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:19.406115] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.415919] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.415980] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=5] ====== check clog disk timer task ====== [2024-06-25 17:00:19.415991] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=7] get_disk_usage(ret=0, capacity(MB):=5529, used(MB):=4371) [2024-06-25 17:00:19.416006] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=6] cannot_recycle_log_size statistics(cannot_recycle_log_size=288379996, threshold=1739461755, need_update_checkpoint_scn=false) [2024-06-25 17:00:19.417282] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=8][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.425652] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:19.426220] INFO [LIB] stat (utility.h:1197) [99517][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=13] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=8, this=0x2b3ae6e6c8f8, extra_info="") [2024-06-25 17:00:19.426230] INFO [LIB] stat (utility.h:1197) [99517][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=1, this=0x2b3ae6e6cb28, extra_info="") [2024-06-25 17:00:19.427880] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [100428][T1003_ReqMemEvi][T1003][Y0-0000000000000000-0-0] [lt=7] run rewrite rule refresh task(rule_mgr_->tenant_id_=1003) [2024-06-25 17:00:19.428223] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100574][T1003_L0_G0][T1003][YB420ABA45B0-00061644F03C36DA-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1008}}) [2024-06-25 17:00:19.428245] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100574][T1003_L0_G0][T1004][YB420ABA45B0-00061644F03C36DA-0-0] [lt=14] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306019138440002, v:0}, arg={tenant_id:1004, ls_id:{id:1008}}, result={tenant_id:1004, ls_id:{id:1008}, cur_readable_scn:{val:1719306019138440002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:19.429927] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.429959] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.429965] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.429967] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.429970] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.432686] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.433416] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF59D3-0-0] [lt=4] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=1333) [2024-06-25 17:00:19.433796] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.436766] WDIAG [SHARE] set_default_timeout_ctx (ob_share_util.cpp:153) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-4012] timeouted(ret=-4012, ret="OB_TIMEOUT", abs_timeout_ts=1719306019200138, ctx_timeout_ts=-1, worker_timeout_ts=1719306019200138, default_timeout=1000000) [2024-06-25 17:00:19.436779] WDIAG [SHARE.LOCATION] batch_renew_ls_locations (ob_ls_location_service.cpp:1236) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=13][errcode=-4012] fail to set default_timeout_ctx(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.436785] WDIAG [SHARE.LOCATION] renew_location_ (ob_ls_location_service.cpp:1005) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] batch renew ls locations failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_ids=[{id:1}]) [2024-06-25 17:00:19.436792] WDIAG [SHARE.LOCATION] get (ob_ls_location_service.cpp:289) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4012] renew location failed(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.436798] WDIAG [SHARE.LOCATION] get_leader (ob_ls_location_service.cpp:328) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] fail to get location(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}) [2024-06-25 17:00:19.436801] WDIAG [SHARE.LOCATION] get_leader (ob_location_service.cpp:82) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] fail to get log stream location leader(ret=-4012, ret="OB_TIMEOUT", cluster_id=1713323324, tenant_id=1001, ls_id={id:1}, leader="0.0.0.0:0", force_renew=true) [2024-06-25 17:00:19.436809] WDIAG [STORAGE.TRANS] fetch_new_range (ob_das_id_rpc.cpp:105) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=7][errcode=-4012] get leader failed(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, GTI_LS={id:1}) [2024-06-25 17:00:19.436814] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4] fetch new DAS ID range finish(ret=-4012, ret="OB_TIMEOUT", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.436819] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] fetch new range failed(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:19.436831] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=1][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:35, local_retry_times:35, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:19.436839] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-4012] failed to close result(close_ret=-4383, ret=-4012) [2024-06-25 17:00:19.436846] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62929][T1_L0_G0][T1002][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:19.436853] INFO [SERVER] process_final (ob_inner_sql_connection.cpp:734) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6] slow inner sql(last_ret=-4012, sql={ObIExecutor:, sql:"SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')"}, process_time=10236341) [2024-06-25 17:00:19.436858] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')"}, aret=-4012, ret=-4012) [2024-06-25 17:00:19.436862] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1002, sql=SELECT tenant_name, database_name, table_name, tablet_id, svr_ip, sql_port, table_id, role, part_num, replica_num, table_type, schema_version, replica_type, dup_replica_type, memstore_percent, spare1, spare2, spare3, spare4, spare5, spare6, complex_table_type, level1_decoded_db_name, level1_decoded_table_name, level2_decoded_db_name, level2_decoded_table_name FROM oceanbase.__all_virtual_proxy_schema WHERE (TENANT_NAME, DATABASE_NAME, TABLE_NAME) = ('ocp', 'ocp_express', 'metric_exporter_address')) [2024-06-25 17:00:19.436917] WDIAG [SERVER] test (ob_query_retry_ctrl.cpp:222) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-5066] execution was terminated(ret=-5066) [2024-06-25 17:00:19.436922] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] [RETRY] check if need retry(v={force_local_retry:false, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-5066}, need_retry=false) [2024-06-25 17:00:19.436931] WDIAG [SERVER] response_result (ob_sync_plan_driver.cpp:120) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=0][errcode=-5066] close result set fail(cret=-4012) [2024-06-25 17:00:19.436941] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:366) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4] sending error packet(ob_error=-5066, client error=-5066, extra_err_info=NULL, lbt()="0x12a0688c 0xa2161e0 0xa1c51c6 0x4e7e290 0x4e35890 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x2b3993985ea5 0x2b3993c9896d") [2024-06-25 17:00:19.436970] INFO [SERVER] send_error_packet (obmp_packet_sender.cpp:549) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6] dump txn free route audit_record(value=5, session->get_sessid()=3221518213, session->get_proxy_sessid()=774413746465638104) [2024-06-25 17:00:19.436977] WDIAG [SERVER] do_process (obmp_query.cpp:823) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=5][errcode=-5066] execute query fail(ret=-5066, timeout_timestamp=1719306019200140) [2024-06-25 17:00:19.436993] WDIAG [SQL.SESSION] set_session_state_ (ob_basic_session_info.cpp:5794) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-5066] session is killed(ret=-5066, sessid_=3221518213, proxy_sessid_=774413746465638104, state=1) [2024-06-25 17:00:19.437004] WDIAG [RPC.OBMYSQL] async_write_data (ob_sql_sock_session.cpp:163) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=6][errcode=-4009] sock has error(ret=-4009) [2024-06-25 17:00:19.437009] WDIAG [SERVER] flush_buffer (obmp_packet_sender.cpp:1032) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=4][errcode=-4009] write response fail(ret=-4009) [2024-06-25 17:00:19.437011] WDIAG [SERVER] process (obmp_query.cpp:402) [62929][T1_L0_G0][T1][YB420ABF4533-0006166133BFE257-0-0] [lt=2][errcode=-5066] failed to flush_buffer(tmp_ret=-4009) [2024-06-25 17:00:19.438267] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=14][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.438793] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABA45AB-000616C80A1FC7F5-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1007}}) [2024-06-25 17:00:19.438808] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABA45AB-000616C80A1FC7F5-0-0] [lt=9] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306019044617003, v:0}, arg={tenant_id:1004, ls_id:{id:1007}}, result={tenant_id:1004, ls_id:{id:1007}, cur_readable_scn:{val:1719306019044617003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:19.446580] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.447300] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=1] will sleep(sleep_us=57000, remain_us=25279876, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=57, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.448180] INFO pktc_resp_cb_on_msg (pktc_resp.h:70) [62234][pnio1][T0][Y0-0000000000000000-0-0] [lt=8] PNIO resp cb not found: packet_id=1035875784 [2024-06-25 17:00:19.449952] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=31240684, this={dio_aligned_buf:{buf_write_offset:614, buf_padding_size:3482, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:50864145, truncate_used_ts:14707412}, log_block_size:67108864, dir_fd:819, io_fd:813}, aligned_buf_len=4096, aligned_buf="ÿÿ", aligned_block_offset=31240192, buf_len=122, total_write_size_=12060, total_write_size_after_dio_=417792, ob_pwrite_used_ts=7556, count=99, each_pwrite_cost=76) [2024-06-25 17:00:19.450170] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F004-0-0] [lt=2] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=96, avg_log_submit_to_slide_time=287) [2024-06-25 17:00:19.450177] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1342, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.454902] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABE88-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b6d5470, extra_info="") [2024-06-25 17:00:19.454987] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=73, this=0x2b3b5b6df3b0, extra_info="palf_id:1008") [2024-06-25 17:00:19.454995] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:19.455014] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3b5b6df5e0, extra_info="palf_id:1008") [2024-06-25 17:00:19.455789] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ACF-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5b6d56a0, extra_info="palf_id:1008") [2024-06-25 17:00:19.455794] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ACF-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=848, this=0x2b3b5b6d58d0, extra_info="palf_id:1008") [2024-06-25 17:00:19.460343] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:213, local_retry_times:213, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:19.460353] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.460356] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:19.460358] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:19.460364] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:19.460521] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABF4538-00061669548FCC03-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301550, send_timestamp=1719305997158970) [2024-06-25 17:00:19.461232] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=8] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-06-25 17:00:19.461257] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=10] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_refresh=false, gts_local_cache={srr:[mts=1719306019132963], gts:1719306019133168000, latest_srr:[mts=1719306019461252]}) [2024-06-25 17:00:19.461586] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62239][pnio1][T0][YB420ABF4533-0006166171AF65D9-0-0] [lt=9][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:19.461595] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62239][pnio1][T0][YB420ABF4533-0006166171AF65D9-0-0] [lt=9] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:19.461606] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62239][pnio1][T0][YB420ABF4533-0006166171AF65D9-0-0] [lt=5] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306019461247]}) [2024-06-25 17:00:19.464305] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=19348804, this={dio_aligned_buf:{buf_write_offset:3509, buf_padding_size:587, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:51289944, truncate_used_ts:14807760}, log_block_size:67108864, dir_fd:825, io_fd:838}, aligned_buf_len=4096, aligned_buf=",ÖoìÜ3¶_æˆ", aligned_block_offset=19345408, buf_len=113, total_write_size_=10840, total_write_size_after_dio_=372736, ob_pwrite_used_ts=6508, count=89, each_pwrite_cost=73) [2024-06-25 17:00:19.466559] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.466567] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.466569] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.466572] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.466574] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.466653] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:19.466790] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-000616432547F006-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305997165246) [2024-06-25 17:00:19.467499] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.470935] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9478332, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=166, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.472297] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=5173044, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=194, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.477526] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62237][pnio1][T1001][YB420ABF4533-00061661718F6536-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.481501] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=1] will sleep(sleep_us=81000, remain_us=22343477, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=81, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.485403] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.486572] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [100471][T1003_LSArchive][T1003][YB420ABF4533-0006164249CE6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:19.493485] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [100481][T1003_HAService][T1003][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:19.497960] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F4-0-0] [lt=4] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:11031, add_timestamp:1719306019497953}) [2024-06-25 17:00:19.497969] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF03-0-0] [lt=25] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=11031, vtable_type={tenant_id:1, type:4}) [2024-06-25 17:00:19.498800] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF03-0-0] [lt=8] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=11031, locations=[{table_id:11031, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:19.498833] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF03-0-0] [lt=32] success to process renew task(task={tenant_id:1, table_id:11031, add_timestamp:1719306019497953}, locations=[{table_id:11031, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}, {table_id:11031, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019498786, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:19.499688] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [99409][T1004_HAService][T1004][Y0-0000000000000000-0-0] [lt=10] start do ls ha handler(ls_id_array_=[{id:1006}, {id:1005}, {id:1007}, {id:1008}, {id:1039}]) [2024-06-25 17:00:19.505237] INFO [LIB] stat (utility.h:1197) [62744][T1_ReplaySrv1][T1][Y0-0000000000000000-0-0] [lt=3] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=9, this=0x2b3a3f4048f8, extra_info="") [2024-06-25 17:00:19.505246] INFO [LIB] stat (utility.h:1197) [62744][T1_ReplaySrv1][T1][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=40, this=0x2b3a3f404b28, extra_info="") [2024-06-25 17:00:19.514193] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:19.514202] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.514213] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2094582, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=214, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.516357] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=14218467, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=135, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.518368] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301556, send_timestamp=1719305997216811) [2024-06-25 17:00:19.518492] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1457398, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=218, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.519141] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.521975] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.525283] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=5] ====== check clog disk timer task ====== [2024-06-25 17:00:19.525291] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=6] get_disk_usage(ret=0, capacity(MB):=230400, used(MB):=0) [2024-06-25 17:00:19.525297] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=4] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=72477573120, need_update_checkpoint_scn=false) [2024-06-25 17:00:19.525746] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.529500] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3175482, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=207, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.529985] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.530017] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=17, this=0x2b3b5a817470, extra_info="") [2024-06-25 17:00:19.530028] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5a884e50, extra_info="id:1005") [2024-06-25 17:00:19.530038] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.530041] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.530043] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.530046] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.530098] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=75, this=0x2b3b5a8213b0, extra_info="palf_id:1005") [2024-06-25 17:00:19.530116] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8215e0, extra_info="palf_id:1005") [2024-06-25 17:00:19.530221] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164325962AD1-0-0] [lt=1][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301545, send_timestamp=1719305997228675) [2024-06-25 17:00:19.530266] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325962AD2-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305997228722) [2024-06-25 17:00:19.530307] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AD2-0-0] [lt=3] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8176a0, extra_info="palf_id:1005") [2024-06-25 17:00:19.530314] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AD2-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=298, this=0x2b3b5a8178d0, extra_info="palf_id:1005") [2024-06-25 17:00:19.531518] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.532740] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.533271] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.534638] INFO [COMMON] replace_fragment_node (ob_kvcache_map.cpp:696) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] Cache replace map node details(ret=0, replace_node_count=0, replace_time=0, replace_start_pos=59900000, replace_num=100000) [2024-06-25 17:00:19.534649] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=10] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=10) [2024-06-25 17:00:19.535901] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [100464][T1003_ReqMemEvi][T1003][Y0-0000000000000000-0-0] [lt=11] sql audit evict task end(request_manager_->get_tenant_id()=1003, evict_high_mem_level=1932735283, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=82982, mem_used=195495936) [2024-06-25 17:00:19.537123] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC061-00061643952B83BC-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=50620681, send_timestamp=1719305968916442) [2024-06-25 17:00:19.538308] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=4][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.541271] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.546788] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F5-0-0] [lt=9][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=1326, pos=316, bitmap=NULL) [2024-06-25 17:00:19.546967] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F5-0-0] [lt=10][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=412, pos=303, bitmap=NULL) [2024-06-25 17:00:19.547168] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F5-0-0] [lt=5][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=76, len=384, pos=368, bitmap=NULL) [2024-06-25 17:00:19.547377] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F5-0-0] [lt=4][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=446, pos=293, bitmap=NULL) [2024-06-25 17:00:19.547573] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F5-0-0] [lt=5][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=421, pos=314, bitmap=NULL) [2024-06-25 17:00:19.550143] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:19.554958] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABE8C-0-0] [lt=3] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1342, avg_group_log_size=122) [2024-06-25 17:00:19.556443] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.558068] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=1] will sleep(sleep_us=58000, remain_us=25169107, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=58, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.559032] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.560268] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-0006164325C300E3-0-0] [lt=5][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301548, send_timestamp=1719305997258719) [2024-06-25 17:00:19.560293] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C300E3-0-0] [lt=11] [PALF STAT GROUP LOG INFO](palf_id=1010, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:19.560374] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1010, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:19.560593] INFO [LIB] stat (utility.h:1197) [63266][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=4] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a872048f8, extra_info="") [2024-06-25 17:00:19.560602] INFO [LIB] stat (utility.h:1197) [63266][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=8] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=1, this=0x2b3a87204b28, extra_info="") [2024-06-25 17:00:19.561295] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.566582] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.566588] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.566591] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.566593] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.566596] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.567102] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.574785] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=10] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289111621632, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:19.576896] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.577627] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62236][pnio1][T1001][YB420ABF4533-00061661718F6537-0-0] [lt=10][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.585457] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.589884] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=11] Cache wash clean map node details(ret=0, clean_node_count=6, clean_time=0, clean_start_pos=172738137, clean_num=402653) [2024-06-25 17:00:19.591853] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F6-0-0] [lt=4][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=73, len=343, pos=314, bitmap=NULL) [2024-06-25 17:00:19.592917] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.594678] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.601499] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF_DUMP](palf_id=1006, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1006, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:96260143, max_lsn:75469139831, max_scn:{val:1719306019461421000, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:75469139831}, reserved_buffer_size:41943040, available_buffer_size:33554432, readable_begin_lsn:{lsn:75427196791}}, last_submit_lsn:{lsn:75469139709}, last_submit_end_lsn:{lsn:75469139831}, last_submit_log_id:96260143, last_submit_log_pid:1, max_flushed_lsn:{lsn:75469139709}, max_flushed_end_lsn:{lsn:75469139831}, max_flushed_log_pid:1, committed_end_lsn:{lsn:75469139831}, last_slide_log_id:96260143, last_slide_scn:{val:1719306019461421000, v:0}, last_slide_lsn:{lsn:75469139709}, last_slide_end_lsn:{lsn:75469139831}, last_slide_log_pid:1, last_slide_log_accum_checksum:3592405039, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2b3a9537a530}, [StateMgr]={this:0x2b3a9537e2b0, self:"10.191.69.51:2882", palf_id:1006, role:"LEADER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, leader:"10.191.69.51:2882", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:96260144, is_changing_config_with_arb:false, reconfirm_start_time_us:1713323710371480, palf_role_change_cb:0x2b3a95384b70, allow_vote:true}, [ConfigMgr]={palf_id:1006, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:1, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"STARTWORKING"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:1}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:1}, persistent_config_version:{proposal_id:1, config_seq:2}, ms_ack_list:0, resend_config_version:{proposal_id:1, config_seq:2}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:-1, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:-1, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2b3a9537bc70}, [ModeMgr]={palf_id:1006, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697225787001, v:0}}, accepted_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697225787001, v:0}}, last_submit_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697225787001, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:1713323710571689, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1006, is_inited:true, min_block_max_scn:{val:1718653213037023001, v:0}, min_block_id:1112, min_block_min_scn:{val:1718597848550157004, v:0}, base_lsn_for_block_gc:{lsn:75425759232}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, log_config_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, log_snapshot_meta:{version:1, base_lsn:{lsn:75425759232}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697225787001, v:0}}}, log_meta_storage:{log_tail:{lsn:3170304}, readable_log_tail:{lsn:3170304}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:75492864000}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1006}, block_mgr:{log_dir:"/data/clog/tenant_1002/1006/meta", dir_fd:815, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:63934464, block_header_serialize_buf_:0x2b3a95380438, flashback_version:0}, log_storage:{log_tail:{lsn:75469139831}, readable_log_tail:{lsn:75469139831}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:75425759232}, min_scn:{val:1719269782386373001, v:0}, curr_block_id:1124, palf_id:1006}, block_mgr:{log_dir:"/data/clog/tenant_1002/1006/log", dir_fd:816, min_block_id:1112, max_block_id:1125, curr_writable_block_id:1124}, logical_block_size_:67104768, curr_block_writable_size_:23724169, block_header_serialize_buf_:0x2b3a953820b8, flashback_version:0}, palf_epoch:1, last_purge_throttling_ts:1713323710370969, this:0x2b3a9537e9f0}, [Reconfirm]={palf_id:1006, self:"10.191.69.51:2882", state:"FINISHED", new_proposal_id:1, prepare_log_ack_list:3"10.191.69.56:2882";"10.189.192.96:2882";"10.186.69.171:2882";, curr_paxos_follower_list:4{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, majority_cnt:3, majority_max_log_server:"10.191.69.56:2882", majority_max_accept_pid:0, majority_max_lsn:{lsn:0}, saved_end_lsn:{lsn:0}, last_submit_prepare_req_time_us:1713323710371613, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2b3a9537e430}) [2024-06-25 17:00:19.601616] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=114] [PALF_DUMP](palf_id=1006, self="10.191.69.51:2882", ack_info_list=[{member:{server:"10.186.69.171:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019467396, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.186.69.176:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019467403, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.189.192.96:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466808, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.191.69.51:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466671, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.191.69.56:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466731, last_flushed_end_lsn:{lsn:75469139831}}]) [2024-06-25 17:00:19.601629] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=11] current log_task status(palf_id=1006, self="10.191.69.51:2882", log_id=96260144, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[{member:{server:"10.186.69.171:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019467396, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.186.69.176:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019467403, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.189.192.96:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466808, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.191.69.51:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466671, last_flushed_end_lsn:{lsn:75469139831}}, {member:{server:"10.191.69.56:2882", timestamp:1, flag:0}, last_ack_time_us:1719306019466731, last_flushed_end_lsn:{lsn:75469139831}}]) [2024-06-25 17:00:19.603478] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [62703][T1_LSMetaCh][T1][YB420ABF4533-0006166172AE6CE2-0-0] [lt=47] fetch root ls success(local_cluster_id=1713323324, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.171:2882", sql_port:2881, role:1, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607828518, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.176:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:4, zone:"zone4", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607829360, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.189.192.96:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:5, zone:"zone5", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607830236, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.51:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607831113, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.56:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["10.186.69.171:2882", "10.191.69.51:2882", "10.191.69.56:2882", "10.186.69.176:2882", "10.189.192.96:2882"]) [2024-06-25 17:00:19.616322] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=1] will sleep(sleep_us=82000, remain_us=22208656, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=82, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.617904] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63299][T1002_LSArchive][T1002][YB420ABF4533-000616423EEE6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:19.618223] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63484][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=8] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:19.624596] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9324671, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=167, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.624613] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=22] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:19.626081] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5019259, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=195, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.626529] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.628554] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.630032] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.630060] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.630065] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.630067] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.630070] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.630275] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABDC060-0006164325962AD6-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301552, send_timestamp=1719305997328723) [2024-06-25 17:00:19.630279] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-0006164325962AD7-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301548, send_timestamp=1719305997328730) [2024-06-25 17:00:19.630328] INFO [LIB] stat (utility.h:1197) [99510][T1004_ApplySrv4][T1004][YB420ABF4533-0006166111AEB555-0-0] [lt=2] [APPLY STAT CB APPEND COST TIME](cur_stat_count=10, stat_interval=5000000, avg (count/cost)=22, this=0x2b3b5a82a280, extra_info="") [2024-06-25 17:00:19.630337] INFO [LIB] stat (utility.h:1197) [99510][T1004_ApplySrv4][T1004][YB420ABF4533-0006166111AEB555-0-0] [lt=9] [APPLY STAT CB IN QUEUE TIME](cur_stat_count=10, stat_interval=5000000, avg (count/cost)=263, this=0x2b3b5a82a4b0, extra_info="") [2024-06-25 17:00:19.630342] INFO [LIB] stat (utility.h:1197) [99510][T1004_ApplySrv4][T1004][YB420ABF4533-0006166111AEB555-0-0] [lt=5] [APPLY STAT CB WAIT COMMIT TIME](cur_stat_count=10, stat_interval=5000000, avg (count/cost)=0, this=0x2b3b5a82a6e0, extra_info="") [2024-06-25 17:00:19.630345] INFO [LIB] stat (utility.h:1197) [99510][T1004_ApplySrv4][T1004][YB420ABF4533-0006166111AEB555-0-0] [lt=3] [APPLY STAT CB EXECUTE TIME](cur_stat_count=10, stat_interval=5000000, avg (count/cost)=-503, this=0x2b3b5a82a910, extra_info="") [2024-06-25 17:00:19.630350] INFO [LIB] stat (utility.h:1197) [99510][T1004_ApplySrv4][T1004][YB420ABF4533-0006166111AEB555-0-0] [lt=4] [APPLY STAT CB TOTAL TIME](cur_stat_count=10, stat_interval=5000000, avg (count/cost)=-217, this=0x2b3b5a82ab40, extra_info="") [2024-06-25 17:00:19.632762] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.638362] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:120) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6] tenant info is invalid(ret=-4076, query_ts=0, standby_scn={val:0, v:0}, this={inited:true, last_id:-1, tenant_id:1004, epoch:-1, self:"10.191.69.51:2882", switch_to_leader_ts:-1}) [2024-06-25 17:00:19.638375] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:134) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=13] ObStandbyTimestampService state(*this={inited:true, last_id:-1, tenant_id:1004, epoch:-1, self:"10.191.69.51:2882", switch_to_leader_ts:-1}) [2024-06-25 17:00:19.638381] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.651736] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [100456][T1003_LCLSender][T1003][Y0-0000000000000000-0-0] [lt=16] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=30358025, total_destructed_detector=30358025, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x2b3b43a382b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-06-25 17:00:19.658637] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.660332] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164325C300E7-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301565, send_timestamp=1719305997358767) [2024-06-25 17:00:19.660359] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C300E7-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1010, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.660640] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABF4538-00061669548FCC0A-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301542, send_timestamp=1719305997359097) [2024-06-25 17:00:19.663083] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.666663] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=19, this=0x2b3a9537b470, extra_info="") [2024-06-25 17:00:19.666674] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=9, avg_log_batch_cnt=1, total_group_log_size=1098, avg_group_log_size=122) [2024-06-25 17:00:19.666681] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT APPEND DATA SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1098) [2024-06-25 17:00:19.666687] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT APPEND COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a9534ee50, extra_info="id:1006") [2024-06-25 17:00:19.666691] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.666695] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.666697] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.666699] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.666703] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.666757] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=79, this=0x2b3a953853b0, extra_info="palf_id:1006") [2024-06-25 17:00:19.666774] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=1098) [2024-06-25 17:00:19.666784] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=4, this=0x2b3a953855e0, extra_info="palf_id:1006") [2024-06-25 17:00:19.666908] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-000616432547F00C-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305997365364) [2024-06-25 17:00:19.666953] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F00C-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a9537b6a0, extra_info="palf_id:1006") [2024-06-25 17:00:19.666961] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F00C-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=303, this=0x2b3a9537b8d0, extra_info="palf_id:1006") [2024-06-25 17:00:19.666964] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F00C-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=24, avg_log_submit_to_flush_time=96, avg_log_submit_to_slide_time=278) [2024-06-25 17:00:19.668087] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1940707, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=215, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.669969] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=59000, remain_us=25057206, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=59, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.670178] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=14064645, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=136, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.672154] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1303737, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=219, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.677655] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62232][pnio1][T1001][YB420ABF4533-00061661718F6538-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:19.683073] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3021908, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=208, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.685524] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.685711] INFO [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:262) [63507][T1002_ArcSender][T1002][YB420ABF4533-00061642411E6725-0-0] [lt=8] ObArchiveSender is running(thread_index=2) [2024-06-25 17:00:19.694252] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.702358] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABF4533-000616604F4F58D5-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620689, send_timestamp=1719305969081668) [2024-06-25 17:00:19.702549] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABF4533-000616604F4F58D5-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620676, send_timestamp=1719305969081872) [2024-06-25 17:00:19.704348] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [100252][T1004_L0_G0][T1004][YB420ABF4533-000616611FFEABF8-0-0] [lt=10] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1004, table_id:11021, add_timestamp:1719306019704341}) [2024-06-25 17:00:19.704355] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF04-0-0] [lt=25] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1004, table_id=11021, vtable_type={tenant_id:1004, type:3}) [2024-06-25 17:00:19.704473] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC088-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a5d1db470, extra_info="") [2024-06-25 17:00:19.704566] INFO [LIB] stat (utility.h:1197) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=12] [PALF STAT WRITE LOG COST TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=76, this=0x2b3a5d1e53b0, extra_info="palf_id:1") [2024-06-25 17:00:19.704579] INFO [LIB] stat (utility.h:1214) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=10] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=6, this=0x2b3a40ffddb0, extra_info="") [2024-06-25 17:00:19.704592] INFO [LIB] stat (utility.h:1197) [62591][T1_LogIOCb0][T1][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=12, stat_interval=1000000, avg (count/cost)=10, this=0x2b3a5d1e55e0, extra_info="palf_id:1") [2024-06-25 17:00:19.705174] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF04-0-0] [lt=7] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1004, table_id=11021, locations=[{table_id:11021, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:19.705200] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF04-0-0] [lt=26] success to process renew task(task={tenant_id:1004, table_id:11021, add_timestamp:1719306019704341}, locations=[{table_id:11021, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}, {table_id:11021, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306019705163, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:19.706850] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABF4533-000616611FFEABF8-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301559, send_timestamp=1719305997405291) [2024-06-25 17:00:19.707026] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100316][T1004_L0_G0][T1004][YB420ABF4533-000616611FFEABF8-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.707048] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABF4533-000616611FFEABF8-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301544, send_timestamp=1719305997405504) [2024-06-25 17:00:19.707199] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABF4533-000616611FFEABF8-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620679, send_timestamp=1719305969086519) [2024-06-25 17:00:19.708503] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [62728][T1_ArcSender][T1][YB420ABF4533-000616423A4E6725-0-0] [lt=10][errcode=-4018] try free send task failed(ret=-4018) [2024-06-25 17:00:19.711325] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99943][T1004_L0_G0][T1004][YB420ABF4533-000616611FFEABF8-0-0] [lt=11][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.711678] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABF4533-000616611FFEABF8-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620678, send_timestamp=1719305969090999) [2024-06-25 17:00:19.726343] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ADA-0-0] [lt=6] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.726603] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F00D-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.726899] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100157][T1004_L0_G0][T1004][YB420ABA45B0-000616443951B6D2-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.730089] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=22, this=0x2bcde0525470, extra_info="") [2024-06-25 17:00:19.730105] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b589fc96e50, extra_info="id:1039") [2024-06-25 17:00:19.730110] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.730140] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.730145] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.730147] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.730150] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.730191] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=36040775, this={dio_aligned_buf:{buf_write_offset:193, buf_padding_size:3903, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:10637072, truncate_used_ts:2870794}, log_block_size:67108864, dir_fd:983, io_fd:846}, aligned_buf_len=4096, aligned_buf="", aligned_block_offset=36040704, buf_len=122, total_write_size_=12294, total_write_size_after_dio_=425984, ob_pwrite_used_ts=6971, count=101, each_pwrite_cost=69) [2024-06-25 17:00:19.730207] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=15] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=71, this=0x2bcde052f3b0, extra_info="palf_id:1039") [2024-06-25 17:00:19.730215] INFO [LIB] stat (utility.h:1214) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbff3b0, extra_info="") [2024-06-25 17:00:19.730223] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=4, this=0x2bcde052f5e0, extra_info="palf_id:1039") [2024-06-25 17:00:19.730364] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ADB-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2bcde05256a0, extra_info="palf_id:1039") [2024-06-25 17:00:19.730370] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962ADB-0-0] [lt=6] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=310, this=0x2bcde05258d0, extra_info="palf_id:1039") [2024-06-25 17:00:19.730373] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABDC060-0006164325962ADC-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301548, send_timestamp=1719305997428824) [2024-06-25 17:00:19.731670] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100202][T1004_L0_G0][T1004][YB420ABA45B0-000616443951B6D2-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.732816] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.732969] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=11][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4013, dropped:279, tid:12536}]) [2024-06-25 17:00:19.734715] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=4] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=9) [2024-06-25 17:00:19.735782] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [63302][T1002_ArcSender][T1002][YB420ABF4533-000616423F1E6725-0-0] [lt=8][errcode=-4018] try free send task failed(ret=-4018) [2024-06-25 17:00:19.738416] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=4][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.741950] INFO pn_ratelimit (group.c:551) [62497][IngressService][T0][Y0-0000000000000000-0-0] [lt=6] PNIO set ratelimit as 9223372036854775807 bytes/s, grp_id=2 [2024-06-25 17:00:19.747890] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=13][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:19.747899] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:19.750336] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62748][T1_ReplaySrv5][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:19.752157] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=83000, remain_us=22072821, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=83, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.757471] INFO [STORAGE.TRANS] post_msg (ob_trans_rpc.cpp:525) [62160][ServerGTimer][T1004][YB420ABF4533-000616616D6E83BA-0-0] [lt=9] post txn free route msg(ret=0, msg={type:81, request_id:741221, req_sess_id:3221632376, tx_sess_id:3221932298, tx_id:{txid:2228768677}, sender:"10.191.69.51:2882", receiver:"10.191.69.56:2882"}) [2024-06-25 17:00:19.757662] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [62160][ServerGTimer][T1004][Y0-0000000000000000-0-0] [lt=14] get_disk_usage(ret=0, capacity(MB):=2073600, used(MB):=1658818) [2024-06-25 17:00:19.760372] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325C300EB-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301544, send_timestamp=1719305997458827) [2024-06-25 17:00:19.760670] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABF4538-00061669548FCC0D-0-0] [lt=14][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301547, send_timestamp=1719305997459122) [2024-06-25 17:00:19.766726] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.766733] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.766735] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.766737] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.766740] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.770350] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [99194][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=8] ====== tenant freeze timer task ====== [2024-06-25 17:00:19.771398] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.771409] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:19.771415] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.771422] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.771428] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.773477] INFO [CLOG.EXTLOG] run1 (ob_cdc_service.cpp:172) [99269][T1004_CdcSrv][T1004][Y0-0000000000000000-0-0] [lt=6] query dest_info_ succ(dest_info=[], archive_dest_changed=false) [2024-06-25 17:00:19.773488] INFO [CLOG.EXTLOG] resize_log_ext_handler_ (ob_cdc_service.cpp:632) [99269][T1004_CdcSrv][T1004][Y0-0000000000000000-0-0] [lt=9] finish to resize log external storage handler(current_ts=1719306019773487, tenant_max_cpu=100, valid_ls_count=0, other_ls_count=0, new_concurrency=0) [2024-06-25 17:00:19.774268] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.774277] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.774283] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:19.774289] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.774294] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.777641] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62235][pnio1][T1001][YB420ABF4533-00061661718F6539-0-0] [lt=9][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:19.778114] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.778123] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.778128] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:19.778135] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.778139] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.778355] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9170912, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=168, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.779746] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=4865594, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=196, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.782865] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=60000, remain_us=24944310, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=60, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.783998] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.784008] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.784013] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:19.784019] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.784024] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.785578] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.788498] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.788508] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.788513] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.788520] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.788525] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.788537] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e69ab20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221632491, assoc_session_id:3221632491, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306019518122, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3a97f681f8, id:3221632491, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e69ab20}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:19.790097] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=8] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289111621632, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:19.793675] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.793682] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.793686] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.793692] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.793696] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.799883] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=23] get_max_applied_scn(scn={val:1713323697230007001, v:0}, this={ls_id_:{id:1009}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713323697230007001, v:0}}) [2024-06-25 17:00:19.799897] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=11] get_min_unreplayed_log_info(lsn={lsn:86651600553}, scn={val:1719306019638250001, v:0}, this={ls_id_:{id:1009}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306019763275, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:86651600553}, next_to_submit_scn_:{val:1719306019638250001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713323697230007001, v:0}, iterator_:{iterator_impl:{buf_:0x2b3ac8f21000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:86651600431}, end_lsn:{lsn:86651600553}, read_buf:{buf_len_:90112, buf_:0x2b3ac8f21000}, block_size:67104768, log_storage_:0x2b3aaf0a3470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306019638250000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306019638250000, v:0}, data_checksum:1616820247, flag:1}}, init_mode_version:0, accumulate_checksum:1174171203, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306019638250000, v:0}}}}}) [2024-06-25 17:00:19.808896] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [62724][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=6] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-06-25 17:00:19.808906] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:99) [62724][T1_TabletShellT][T1][Y0-0000000000000000-0-0] [lt=6] [emptytablet] succeed to change tablet to empty shell(ret=0, ret="OB_SUCCESS", ls_cnt=1, times=4154) [2024-06-25 17:00:19.811462] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.811470] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.811474] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.811480] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.811484] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.821319] INFO do_work (ob_rl_mgr.cpp:709) [62262][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=8] swc wakeup.(stat_period_=1000000, ready=false) [2024-06-25 17:00:19.821993] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1786802, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=216, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.822729] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [62723][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=5] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-06-25 17:00:19.822741] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [62723][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=9] [tabletchange] task check ls(ls->get_ls_id()={id:1}, tablet_persist_trigger=0) [2024-06-25 17:00:19.822748] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:228) [62723][T1_ObTimer][T1][Y0-0000000000000000-0-0] [lt=5] [tabletchange] succeed to gc_tablet(ret=0, ret="OB_SUCCESS", ls_cnt=1, times=4081) [2024-06-25 17:00:19.824048] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=13910775, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=137, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.824064] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.824073] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.824079] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.824085] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.824091] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.825950] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1149940, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=220, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.829933] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [62719][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=11] get_min_unreplayed_log_info(lsn={lsn:35325765024}, scn={val:1719306019747657001, v:0}, this={ls_id_:{id:1}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306019750266, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:35325765024}, next_to_submit_scn_:{val:1719306019747657001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1, v:0}, iterator_:{iterator_impl:{buf_:0x2b3a6c849000, next_round_pread_size:3678208, curr_read_pos:539, curr_read_buf_start_pos:0, curr_read_buf_end_pos:539, log_storage_:{IteratorStorage:{start_lsn:{lsn:35325764485}, end_lsn:{lsn:35325765024}, read_buf:{buf_len_:708608, buf_:0x2b3a6c849000}, block_size:67104768, log_storage_:0x2b3a5d1e1470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306019747657000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:451, scn_:{val:1719306019747657000, v:0}, data_checksum:2836757861, flag:0}}, init_mode_version:0, accumulate_checksum:3568790969, curr_entry_is_padding:0, padding_entry_size:483, padding_entry_scn:{val:1719306019747657000, v:0}}}}}) [2024-06-25 17:00:19.830144] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.830176] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.830181] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.830183] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.830186] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.830223] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=12] [PALF STAT COMMITTED LOG SIZE](palf_id=1039, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:19.830241] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABF4533-0006166171EF486D-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301546, send_timestamp=1719305997528694) [2024-06-25 17:00:19.830388] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164325962AE1-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301542, send_timestamp=1719305997528845) [2024-06-25 17:00:19.830389] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164325962AE0-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301550, send_timestamp=1719305997528839) [2024-06-25 17:00:19.830440] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AE0-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1039, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=28, avg_log_submit_to_flush_time=90, avg_log_submit_to_slide_time=284) [2024-06-25 17:00:19.830478] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AE1-0-0] [lt=10] [PALF STAT LOG TASK TIME](palf_id=1005, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=20, avg_log_submit_to_flush_time=90, avg_log_submit_to_slide_time=277) [2024-06-25 17:00:19.832870] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.833066] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4012, dropped:25, tid:62821}]) [2024-06-25 17:00:19.833436] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF59E3-0-0] [lt=5] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:19.836658] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:920) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=8] can close safely, do destroy(*s={this:0x2b7098196d70, session_id:3221518213, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:256, sql_initiative_shutdown:false, reader:{fd:927}, err:5, last_decode_time:1719306009199675, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:686, pending_flag:1, may_handling_flag:false, handler_close_flag:false}) [2024-06-25 17:00:19.836681] INFO [RPC.OBMYSQL] sm_conn_unlock_tenant (obsm_conn_callback.cpp:159) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=17] unlock session of tenant(conn.sessid_=3221518213, proxy_sessid=774413746465638104, conn.tenant_id_=1) [2024-06-25 17:00:19.836696] WDIAG [COMMON] get (ob_link_hashmap.h:313) [62929][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=3][errcode=0] uref leak check(node->uref_=1077129732, lbt()="0x12a0688c 0x554c90b 0xe13e893 0xe13fff0 0x521f0a3 0x51ffdf7 0xa20f876 0x4e24eed 0x4e13175 0x9f683b4 0x12d2c90c 0x2b3993985ea5 0x2b3993c9896d") [2024-06-25 17:00:19.836689] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=7] connection close(sessid=3221518213, proxy_sessid=774413746465638104, tenant_id=1, server_id=1, from_proxy=true, from_java_client=false, c/s protocol="OB_2_0_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=true, ret=0, trace_id=YB420ABF4533-0006166133BFE257-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:4, cur_pkt_pos_:6, pkt_rec[0]:{send:obp_mysql_header_{len_:6342, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[1]:{send:obp20_header_:{payload_len:14, pkt_seq:0, req_id:2398505, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:539, seq_:2}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[2]:{send:obp20_header_:{payload_len:43, pkt_seq:0, req_id:2398506, flag_.flags:42, ext_flags:0}, obp_mysql_header_{len_:401, seq_:2}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}, pkt_rec[3]:{receive:obp20_header_:{payload_len:248, pkt_seq:0, req_id:2398507, flag_.flags:62, ext_flags:0}, obp_mysql_header_{len_:244, rec_:248, seq_:0}, pkt_name:"COM_QUERY", obp_mysql_header_.type_:3, obp_mysql_header_.is_send_:0}, pkt_rec[4]:{send:obp_compess_header_{com_len:135, com_seq:1}, obp20_header_:{payload_len:107, pkt_seq:1, req_id:2398507, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:32, seq_:2}, pkt_name:"PKT_ERR", obp_mysql_header_.is_send_:1}, pkt_rec[5]:{send:obp_compess_header_{com_len:135, com_seq:1}, obp20_header_:{payload_len:107, pkt_seq:1, req_id:2398507, flag_.flags:10, ext_flags:0}, obp_mysql_header_{len_:75, seq_:3}, pkt_name:"PKT_OKP", obp_mysql_header_.is_send_:1}], disconnect_state=4) [2024-06-25 17:00:19.836728] INFO [SERVER] run (obmp_disconnect.cpp:82) [62929][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=7] free session successfully(sessid=3221518213) [2024-06-25 17:00:19.836734] INFO [SERVER] run (obmp_disconnect.cpp:89) [62929][T1_L0_G0][T1][Y0-0000000000000000-0-0] [lt=6] mark session id unused(sessid=3221518213) [2024-06-25 17:00:19.836840] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=2868141, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=209, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.838470] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.845227] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.845240] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=13][errcode=-4013] [2024-06-25 17:00:19.845245] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.845251] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.845255] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.847812] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-00061643246151E9-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5407, fly_ts=22301547, send_timestamp=1719305997546265) [2024-06-25 17:00:19.849991] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:19.850303] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABF4533-0006166172BEBE9A-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301552, send_timestamp=1719305997548751) [2024-06-25 17:00:19.855991] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AE3-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=89, avg_log_submit_to_slide_time=850) [2024-06-25 17:00:19.856000] INFO [PALF] read (log_cache.cpp:100) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=33] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.860420] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325C300EF-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301545, send_timestamp=1719305997558875) [2024-06-25 17:00:19.860712] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABF4538-00061669548FCC11-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301547, send_timestamp=1719305997559165) [2024-06-25 17:00:19.860733] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABF4538-00061669548FCC11-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1010, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=91, avg_log_submit_to_slide_time=287) [2024-06-25 17:00:19.860739] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT HOT CACHE HIT RATE](palf_id=1010, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:19.862496] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B4D-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1009, self="10.191.69.51:2882", committed size=967) [2024-06-25 17:00:19.862573] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=967) [2024-06-25 17:00:19.866654] INFO [CLOG] runTimerTask (ob_log_replay_service.cpp:159) [100441][T1003_ReplayPro][T1003][Y0-0000000000000000-0-0] [lt=10] dump tenant replay process(tenant_id=1003, unreplayed_log_size(MB)=0, estimate_time(second)=0, replayed_log_size(MB)=0, last_replayed_log_size(MB)=0, round_cost_time(second)=10, pending_replay_log_size(MB)=0) [2024-06-25 17:00:19.866775] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.866782] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.866784] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.866786] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.866790] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.874966] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164324713061-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=22301542, send_timestamp=1719305997573424) [2024-06-25 17:00:19.877673] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62245][pnio1][T1001][YB420ABF4533-00061661718F653A-0-0] [lt=8][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:19.878396] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:19.878407] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:19.878413] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999946, timeout_timestamp=1719306049877938) [2024-06-25 17:00:19.878420] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.878422] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:19.878427] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:19.878432] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:19.878435] WDIAG [SQL] open (ob_result_set.cpp:157) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:19.878438] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:19.878442] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:19.878447] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:19.878454] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=6][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:19.878461] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:19.878465] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:19.878467] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:19.878473] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:19.878479] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4013, ret=-4013) [2024-06-25 17:00:19.878484] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.878489] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:19.878492] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=2][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:19.878495] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] query failed(ret=-4013, conn=0x2b3a78d44050, start=1719306019877938, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.878502] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=6][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:19.878505] WDIAG [SHARE.PT] get_by_tenant (ob_persistent_ls_table.cpp:639) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=2][errcode=-4013] execute sql failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:19.878530] WDIAG [SHARE.PT] get_by_tenant (ob_ls_table_operator.cpp:252) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] get all ls info by persistent_ls_ failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002) [2024-06-25 17:00:19.878535] WDIAG [SHARE] inner_open_ (ob_ls_table_iterator.cpp:104) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] fail to get ls infos by tenant(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, inner_table_only=false) [2024-06-25 17:00:19.878539] WDIAG [SHARE] next (ob_ls_table_iterator.cpp:71) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] fail to open iterator(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.878543] WDIAG [SERVER] build_replica_map_ (ob_tenant_meta_checker.cpp:334) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=4][errcode=-4013] ls table iterator next failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.878547] WDIAG [SERVER] check_ls_table_ (ob_tenant_meta_checker.cpp:214) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] build replica map from ls table failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", mode=0) [2024-06-25 17:00:19.878553] WDIAG [SERVER] check_ls_table (ob_tenant_meta_checker.cpp:188) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=5][errcode=-4013] check ls table failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", mode=0) [2024-06-25 17:00:19.878556] WDIAG [SERVER] runTimerTask (ob_tenant_meta_checker.cpp:44) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE858-0-0] [lt=3][errcode=-4013] fail to check ls meta table(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.880360] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABF4533-0006166172BEBE9F-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301550, send_timestamp=1719305997578809) [2024-06-25 17:00:19.885624] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.889030] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=84000, remain_us=21935949, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=84, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:19.896753] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=61000, remain_us=24830423, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=61, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:19.897090] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99938][T1004_L0_G0][T1004][YB420ABA45B1-0006164629D475BC-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.899929] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=22][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-06-25 17:00:19.899936] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=6] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-06-25 17:00:19.899995] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.900004] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.900009] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.900016] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.900021] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.901674] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=16] re_register_parent reason(palf_id=1008, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306015899493, last_first_register_time_us=1713323715481338, register_time_us=1719306015899493, curr_time_us=1719306019901673) [2024-06-25 17:00:19.901699] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=11] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882", reason=1, leader="10.186.69.171:2882", register_time_us=1719306019901686, last_submit_register_req_time_us=1719306019901686, register_parent_reason=1) [2024-06-25 17:00:19.901708] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=9] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882") [2024-06-25 17:00:19.902187] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99932][T1004_L0_G0][T1004][YB420ABA45B1-0006164629D475BC-0-0] [lt=8][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:19.902874] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.902882] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.902886] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:19.902892] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.902896] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.904596] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC08C-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=3427, avg_group_log_size=311) [2024-06-25 17:00:19.904684] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=5] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=3427) [2024-06-25 17:00:19.905427] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A089-0-0] [lt=10] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a5d1db6a0, extra_info="palf_id:1") [2024-06-25 17:00:19.905434] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A089-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=833, this=0x2b3a5d1db8d0, extra_info="palf_id:1") [2024-06-25 17:00:19.906847] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.906855] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.906860] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:19.906864] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.906870] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.911075] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.911084] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.911089] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:19.911096] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.911100] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.920724] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.920734] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.920739] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.920746] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.920750] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.922717] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [62718][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=11] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=969588866, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=408846, mem_used=767425536) [2024-06-25 17:00:19.930201] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:19.930238] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1005, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:19.930247] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:19.930251] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.930253] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.930258] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.930445] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-0006164325962AE5-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301549, send_timestamp=1719305997628895) [2024-06-25 17:00:19.932021] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9017246, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=169, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:19.932926] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:19.933395] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=4711946, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=197, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:19.933624] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.933633] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.933639] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.933645] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.933650] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.934780] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=8) [2024-06-25 17:00:19.936491] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.936499] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:19.936503] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.936509] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.936513] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.938524] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.938591] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.938602] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:19.938608] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.938615] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.938619] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.938808] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [99277][T1004_LockWaitM][T1004][Y0-0000000000000000-0-0] [lt=8] report RowHolderMapper summary info(count=4, bkt_cnt=248) [2024-06-25 17:00:19.944366] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.944375] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.944380] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.944386] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.944390] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.944809] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [42764][T1002_ComTimerT][T1002][Y0-0000000000000000-0-0] [lt=8] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-06-25 17:00:19.950358] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=7] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:19.954139] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.954148] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:19.954153] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.954159] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.954163] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.960484] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325C300F3-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301546, send_timestamp=1719305997658937) [2024-06-25 17:00:19.960508] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C300F3-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf16f470, extra_info="") [2024-06-25 17:00:19.960602] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=77, this=0x2b3aaf1793b0, extra_info="palf_id:1010") [2024-06-25 17:00:19.960611] INFO [LIB] stat (utility.h:1214) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=5, this=0x2b3a8ce1d3b0, extra_info="") [2024-06-25 17:00:19.960621] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3aaf1795e0, extra_info="palf_id:1010") [2024-06-25 17:00:19.960805] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC8FB-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf16f6a0, extra_info="palf_id:1010") [2024-06-25 17:00:19.960810] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC8FB-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=288, this=0x2b3aaf16f8d0, extra_info="palf_id:1010") [2024-06-25 17:00:19.961748] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=8] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_refresh=false, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306019961741]}) [2024-06-25 17:00:19.961931] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=11] gts statistics(tenant_id=1004, gts_rpc_cnt=10, get_gts_cache_cnt=454, get_gts_with_stc_cnt=0, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-06-25 17:00:19.962050] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62240][pnio1][T0][YB420ABF4533-0006166171AF65DE-0-0] [lt=7][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:19.962060] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62240][pnio1][T0][YB420ABF4533-0006166171AF65DE-0-0] [lt=10] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:19.962074] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62240][pnio1][T0][YB420ABF4533-0006166171AF65DE-0-0] [lt=7] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306019961741]}) [2024-06-25 17:00:19.966837] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:19.966844] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:19.966847] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:19.966849] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:19.966852] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:19.967057] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-000616432547F018-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305997665513) [2024-06-25 17:00:19.969608] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [99254][T1004_ReqMemEvi][T1004][Y0-0000000000000000-0-0] [lt=10] run rewrite rule refresh task(rule_mgr_->tenant_id_=1004) [2024-06-25 17:00:19.971943] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:19.971952] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:19.971957] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:19.971963] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:19.971967] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:19.975878] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1632916, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=217, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:19.977776] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62239][pnio1][T1001][YB420ABF4533-00061661718F653B-0-0] [lt=9][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:19.977903] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=13756920, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=138, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:19.979714] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=996176, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=221, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:19.981218] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=14] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1004, server_version={version:{val:1719306019370955004, v:0}, total_part_count:5, valid_inner_part_count:0, valid_user_part_count:5}, server_version_delta=610261, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="10.191.69.52:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=58348883, succ_cluster_heartbeat_count=58348883, cluster_heartbeat_interval=50000, local_cluster_version={val:1719305752101344002, v:0}, local_cluster_delta=267879872, force_self_check=false, weak_read_refresh_interval=100000) [2024-06-25 17:00:19.985680] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:19.987741] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=3] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1) [2024-06-25 17:00:19.987753] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=10] start lib cache evict(tenant_id=1, mem_hold=312475648, mem_limit=2019976805, cache_obj_num=914, cache_node_num=913) [2024-06-25 17:00:19.987760] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=6] end lib cache evict(tenant_id=1, cache_evict_num=0, mem_hold=312475648, mem_limit=2019976805, cache_obj_num=914, cache_node_num=913) [2024-06-25 17:00:19.987765] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=4] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:19.990321] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=12] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289109524480, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:19.990477] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2714504, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=210, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:19.990038] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=5] Dumping All Cache Objs(alloc_obj_list.count()=914, alloc_obj_list=[{obj_id:1, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:2, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:97, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176332}, {obj_id:98, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276716}, {obj_id:99, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:112, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:737569, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219837}, {obj_id:274, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303715}, {obj_id:275, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:248080}, {obj_id:276, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271604}, {obj_id:277, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216033}, {obj_id:278, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195958}, {obj_id:279, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183659}, {obj_id:280, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176078}, {obj_id:281, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196297}, {obj_id:1180050, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209023}, {obj_id:282, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:283, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:284, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179794}, {obj_id:285, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230228}, {obj_id:286, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188271}, {obj_id:287, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196930}, {obj_id:288, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174576}, {obj_id:289, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301933}, {obj_id:290, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211414}, {obj_id:291, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224502}, {obj_id:292, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190726}, {obj_id:293, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202255}, {obj_id:294, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199095}, {obj_id:295, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199093}, {obj_id:296, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193076}, {obj_id:297, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214118}, {obj_id:298, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199546}, {obj_id:299, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199766}, {obj_id:301, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190029}, {obj_id:303, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186863}, {obj_id:304, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185594}, {obj_id:306, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:307, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200193}, {obj_id:308, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193157}, {obj_id:309, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188399}, {obj_id:310, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185761}, {obj_id:311, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:312, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185793}, {obj_id:313, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:314, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185738}, {obj_id:315, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192013}, {obj_id:316, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187261}, {obj_id:317, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183532}, {obj_id:318, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210345}, {obj_id:319, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183340}, {obj_id:320, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188170}, {obj_id:321, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183730}, {obj_id:322, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191888}, {obj_id:323, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183436}, {obj_id:324, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184586}, {obj_id:325, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188962}, {obj_id:326, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:308297}, {obj_id:327, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:251495}, {obj_id:329, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282230}, {obj_id:330, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225422}, {obj_id:389, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183962}, {obj_id:390, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182906}, {obj_id:391, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:392, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:394, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:395, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:396, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:398, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:399, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:400, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:402, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:404, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178712}, {obj_id:482, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214699}, {obj_id:483, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193516}, {obj_id:484, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200871}, {obj_id:485, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201183}, {obj_id:486, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200215}, {obj_id:487, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202195}, {obj_id:490, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191892}, {obj_id:491, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:492, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:493, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:494, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:495, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:496, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:498, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:501, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179663}, {obj_id:503, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185730}, {obj_id:504, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:505, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157686}, {obj_id:510, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:515, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157686}, {obj_id:518, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157704}, {obj_id:519, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:521, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178056}, {obj_id:522, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176744}, {obj_id:523, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174432}, {obj_id:524, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214695}, {obj_id:525, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178793}, {obj_id:526, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:528, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207447}, {obj_id:529, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:275361}, {obj_id:530, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179806}, {obj_id:531, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179440}, {obj_id:532, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191447}, {obj_id:533, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:275362}, {obj_id:534, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218570}, {obj_id:535, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214216}, {obj_id:536, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208800}, {obj_id:543, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276362}, {obj_id:544, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181654}, {obj_id:545, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192407}, {obj_id:546, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:547, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:307440}, {obj_id:548, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:549, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:550, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:551, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:552, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276714}, {obj_id:553, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181974}, {obj_id:554, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192767}, {obj_id:555, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:556, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:557, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:558, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:559, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:560, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:562, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:565, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205804}, {obj_id:566, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197099}, {obj_id:567, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191367}, {obj_id:568, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177328}, {obj_id:569, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182496}, {obj_id:570, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:571, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:572, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180277}, {obj_id:574, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:575, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181673}, {obj_id:577, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196012}, {obj_id:578, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186809}, {obj_id:580, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191323}, {obj_id:581, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:582, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:586, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:589, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214431}, {obj_id:595, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:607, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:608, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201420}, {obj_id:609, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:610, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:611, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:612, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188872}, {obj_id:613, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:299219}, {obj_id:614, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182108}, {obj_id:616, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179156}, {obj_id:617, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173136}, {obj_id:618, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184752}, {obj_id:619, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186441}, {obj_id:620, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179449}, {obj_id:621, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178725}, {obj_id:622, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223877}, {obj_id:623, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:624, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:626, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:627, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197179}, {obj_id:630, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184682}, {obj_id:631, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183630}, {obj_id:638, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196470}, {obj_id:641, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:642, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282175}, {obj_id:643, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192056}, {obj_id:644, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186353}, {obj_id:645, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180529}, {obj_id:646, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173811}, {obj_id:647, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172864}, {obj_id:658, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157686}, {obj_id:659, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157711}, {obj_id:668, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:669, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:672, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:673, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:680, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157781}, {obj_id:696, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199763}, {obj_id:697, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199763}, {obj_id:698, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179616}, {obj_id:699, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179616}, {obj_id:700, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192472}, {obj_id:701, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173811}, {obj_id:703, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199763}, {obj_id:704, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199763}, {obj_id:705, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199763}, {obj_id:706, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174869}, {obj_id:714, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:715, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:716, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:166010}, {obj_id:717, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:162314}, {obj_id:718, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157829}, {obj_id:719, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:724, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178368}, {obj_id:725, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:159523}, {obj_id:727, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194538}, {obj_id:730, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176677}, {obj_id:731, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:305791}, {obj_id:732, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185321}, {obj_id:733, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174560}, {obj_id:734, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172752}, {obj_id:737, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:739, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196384}, {obj_id:740, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173852}, {obj_id:744, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182369}, {obj_id:752, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175760}, {obj_id:753, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180460}, {obj_id:755, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176751}, {obj_id:771, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185223}, {obj_id:772, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184511}, {obj_id:781, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:786, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210446}, {obj_id:787, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:825, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1180787, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207498}, {obj_id:1180962, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303379}, {obj_id:1454, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193581}, {obj_id:50733, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193543}, {obj_id:2399, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221425}, {obj_id:2423, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2447, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2528, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181621}, {obj_id:1182546, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221553}, {obj_id:641850, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:333444}, {obj_id:1182580, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1182617, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:258305}, {obj_id:1182618, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179635}, {obj_id:789398, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205871}, {obj_id:1182707, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193403}, {obj_id:150468, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196073}, {obj_id:1183028, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:262574}, {obj_id:397215, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193944}, {obj_id:839774, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175536}, {obj_id:1086494, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:220700}, {obj_id:693718, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:306743}, {obj_id:841223, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218155}, {obj_id:251559, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:573738}, {obj_id:251564, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:995399}, {obj_id:251572, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186890}, {obj_id:251575, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229551}, {obj_id:251578, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292553}, {obj_id:251595, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1136868, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189832}, {obj_id:1136869, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189565}, {obj_id:1136870, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177216}, {obj_id:399522, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:353488}, {obj_id:1137043, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188840}, {obj_id:1087936, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245307}, {obj_id:1087937, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:248426}, {obj_id:1087944, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:335259}, {obj_id:1087945, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:385283}, {obj_id:1087948, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:398171}, {obj_id:1087949, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:481681}, {obj_id:1087950, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:413928}, {obj_id:1087951, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:505577}, {obj_id:1087952, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:402727}, {obj_id:1087953, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:490052}, {obj_id:1087954, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:373499}, {obj_id:1087955, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:444164}, {obj_id:1087958, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:355472}, {obj_id:1087959, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:418640}, {obj_id:793660, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:330128}, {obj_id:793662, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276666}, {obj_id:155139, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:548658, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197785}, {obj_id:893070, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219019}, {obj_id:745601, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:326781}, {obj_id:696557, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199066}, {obj_id:8498, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:8499, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:500677}, {obj_id:8500, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190714}, {obj_id:8501, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223442}, {obj_id:8510, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176371}, {obj_id:8514, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177545}, {obj_id:8515, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172928}, {obj_id:8517, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204160}, {obj_id:8518, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188778}, {obj_id:8519, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184970}, {obj_id:8521, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:8522, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185752}, {obj_id:8525, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174043}, {obj_id:8527, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174080}, {obj_id:8529, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175223}, {obj_id:8530, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191631}, {obj_id:8531, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:8532, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177270}, {obj_id:8533, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188473}, {obj_id:8534, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188530}, {obj_id:8536, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175596}, {obj_id:8537, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176756}, {obj_id:8538, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179720}, {obj_id:8539, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173811}, {obj_id:8540, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179286}, {obj_id:8541, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184717}, {obj_id:8542, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183168}, {obj_id:8543, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191567}, {obj_id:8544, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203564}, {obj_id:8545, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213399}, {obj_id:8550, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179898}, {obj_id:8551, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:8552, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:8553, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:8554, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172528}, {obj_id:8556, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193964}, {obj_id:8559, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172160}, {obj_id:8569, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181013}, {obj_id:8570, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178561}, {obj_id:8571, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179597}, {obj_id:8572, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181930}, {obj_id:8573, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191549}, {obj_id:8574, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185675}, {obj_id:8575, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271426}, {obj_id:8577, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:255998}, {obj_id:8578, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232003}, {obj_id:8579, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301272}, {obj_id:8580, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234280}, {obj_id:8582, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202310}, {obj_id:8590, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182942}, {obj_id:8592, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202971}, {obj_id:8593, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182392}, {obj_id:8594, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218911}, {obj_id:8595, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182448}, {obj_id:8598, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204268}, {obj_id:8599, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179164}, {obj_id:8608, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194888}, {obj_id:8613, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191819}, {obj_id:696958, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192822}, {obj_id:8790, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174539}, {obj_id:1188723, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:296704}, {obj_id:1188736, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183934}, {obj_id:1188737, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:265701}, {obj_id:1188738, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:305908}, {obj_id:1188739, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256316}, {obj_id:1188740, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:299497}, {obj_id:1139773, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231817}, {obj_id:1139774, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227447}, {obj_id:1139775, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234072}, {obj_id:1139776, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231307}, {obj_id:1139777, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240818}, {obj_id:1139778, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242250}, {obj_id:1139779, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:236311}, {obj_id:1139780, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235035}, {obj_id:1139781, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:243082}, {obj_id:1139782, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247885}, {obj_id:1139783, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238568}, {obj_id:1139784, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238553}, {obj_id:1139786, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:310506}, {obj_id:1139787, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:347917}, {obj_id:1139788, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:387000}, {obj_id:1139789, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:465236}, {obj_id:1139790, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:440971}, {obj_id:1139791, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:545609}, {obj_id:1139792, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:454412}, {obj_id:1139793, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:563727}, {obj_id:1139794, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:292497}, {obj_id:1139795, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:323145}, {obj_id:1139796, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:416211}, {obj_id:1139797, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:509604}, {obj_id:9298, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175422}, {obj_id:107718, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200349}, {obj_id:9516, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174576}, {obj_id:1140302, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:285958}, {obj_id:1140303, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320103}, {obj_id:993262, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179166}, {obj_id:993368, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179331}, {obj_id:993390, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179169}, {obj_id:993440, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179298}, {obj_id:551520, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188170}, {obj_id:551523, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:568093}, {obj_id:551525, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187522}, {obj_id:551529, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196997}, {obj_id:551533, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1027006}, {obj_id:551546, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:393312}, {obj_id:551558, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:650014, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192296}, {obj_id:1191636, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:263303}, {obj_id:1191637, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276482}, {obj_id:1191638, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:368939}, {obj_id:1191639, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:436931}, {obj_id:1191640, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:371228}, {obj_id:1191641, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:442099}, {obj_id:1191642, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:377976}, {obj_id:1191643, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:450423}, {obj_id:1191644, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:328494}, {obj_id:1191645, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:374846}, {obj_id:1191646, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:330766}, {obj_id:1191647, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:381562}, {obj_id:1191648, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297098}, {obj_id:1191649, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:330767}, {obj_id:1191650, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279006}, {obj_id:1191651, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:300721}, {obj_id:1191652, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:309107}, {obj_id:1191653, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:319929}, {obj_id:1191654, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:249785}, {obj_id:1191655, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:254408}, {obj_id:1191656, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205023}, {obj_id:1142646, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:293344}, {obj_id:651095, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:269588}, {obj_id:651096, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232922}, {obj_id:995247, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286678}, {obj_id:995248, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198552}, {obj_id:995249, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199145}, {obj_id:995250, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279990}, {obj_id:995251, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179330}, {obj_id:995271, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180091}, {obj_id:995272, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181318}, {obj_id:946365, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207965}, {obj_id:897362, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184764}, {obj_id:996721, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217721}, {obj_id:849254, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:323413}, {obj_id:407282, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:570916}, {obj_id:407286, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1001046}, {obj_id:407289, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1166431}, {obj_id:407294, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1032702}, {obj_id:407298, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:258169}, {obj_id:407299, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191773}, {obj_id:407308, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:112756, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260624}, {obj_id:112757, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:264190}, {obj_id:112758, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177088}, {obj_id:112759, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181113}, {obj_id:112760, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179110}, {obj_id:112761, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176843}, {obj_id:112766, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242573}, {obj_id:112767, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192979}, {obj_id:112768, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192408}, {obj_id:112769, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:284099}, {obj_id:112770, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174589}, {obj_id:112771, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256377}, {obj_id:112772, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176247}, {obj_id:112773, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:269247}, {obj_id:112774, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241866}, {obj_id:112775, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195217}, {obj_id:112776, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:112777, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:112782, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:938545}, {obj_id:112787, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:269947}, {obj_id:112788, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1828995}, {obj_id:112789, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:632776}, {obj_id:112793, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197485}, {obj_id:112794, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:505900}, {obj_id:112795, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:306517}, {obj_id:112796, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209204}, {obj_id:112800, tenant_id:1, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199028}, {obj_i [2024-06-25 17:00:19.990576] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [62614][T1_PlanCacheEvi][T1][Y0-0000000000000000-0-0] [lt=537] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:20.005757] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.005766] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.005770] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:20.005776] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.005780] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.011648] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=62000, remain_us=24715528, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=62, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:20.021246] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [63452][T1001_LockWaitM][T1001][Y0-0000000000000000-0-0] [lt=9] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-06-25 17:00:20.022383] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [62691][T1_ReplayProces][T1][Y0-0000000000000000-0-0] [lt=7] get_replay_process success(id={id:1}, replayed_log_size=35325765685, unreplayed_log_size=0) [2024-06-25 17:00:20.022404] INFO [CLOG] runTimerTask (ob_log_replay_service.cpp:159) [62691][T1_ReplayProces][T1][Y0-0000000000000000-0-0] [lt=9] dump tenant replay process(tenant_id=1, unreplayed_log_size(MB)=0, estimate_time(second)=0, replayed_log_size(MB)=33689, last_replayed_log_size(MB)=33689, round_cost_time(second)=10, pending_replay_log_size(MB)=0) [2024-06-25 17:00:20.023060] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [100346][T1003_PalfGC][T1003][Y0-0000000000000000-0-0] [lt=20] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):230400, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):230400, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):0, sequence:1}) [2024-06-25 17:00:20.025738] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=21] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:20.026469] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AE9-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=97, avg_log_submit_to_slide_time=837) [2024-06-25 17:00:20.026475] INFO [PALF] read (log_cache.cpp:100) [99521][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=12] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:20.026853] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=85000, remain_us=21798125, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=85, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:20.027458] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63474][T1001_LSArchive][T1001][YB420ABF4533-000616423F8E6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:20.027720] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99934][T1004_L0_G0][T1004][YB420ABF4538-0006166915BFA5CD-0-0] [lt=8][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.030257] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] [PALF STAT GROUP LOG INFO](palf_id=1039, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:20.030267] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT APPEND DATA SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1220) [2024-06-25 17:00:20.030275] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.030303] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT APPEND DATA SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1220) [2024-06-25 17:00:20.030309] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.030312] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.030315] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.030318] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.030388] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:20.030528] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-0006164325962AEA-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301554, send_timestamp=1719305997728974) [2024-06-25 17:00:20.030536] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325962AEB-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305997728992) [2024-06-25 17:00:20.031906] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100139][T1004_L0_G0][T1004][YB420ABF4538-0006166915BFA5CD-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.032983] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.033522] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1342) [2024-06-25 17:00:20.036496] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.036506] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:20.036511] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:20.036516] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.036523] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.038579] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.040920] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.040929] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.040933] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:20.040940] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.040945] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.050196] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F01A-0-0] [lt=2] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:20.052883] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [99392][T1004_ReqMemEvi][T1004][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1004, evict_high_mem_level=20669530113, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=8345583, mem_used=15148855296) [2024-06-25 17:00:20.053519] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63292][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=9] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7278, mem_used=12478464) [2024-06-25 17:00:20.055320] INFO [LIB] stat (utility.h:1214) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=7, this=0x2b3afcbfec70, extra_info="") [2024-06-25 17:00:20.060536] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325C300F7-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301548, send_timestamp=1719305997758987) [2024-06-25 17:00:20.060877] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABF4533-00061661728FC8FF-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301550, send_timestamp=1719305997759327) [2024-06-25 17:00:20.062619] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B55-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1009, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=8, avg_log_batch_cnt=0, total_group_log_size=967, avg_group_log_size=120) [2024-06-25 17:00:20.063454] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F01B-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1009, self="10.191.69.51:2882", total_slide_log_cnt=8, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=96, avg_log_submit_to_slide_time=839) [2024-06-25 17:00:20.063458] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=14] [PALF STAT HOT CACHE HIT RATE](palf_id=1009, read_size=967, hit_cnt=8, read_cnt=8, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:20.066893] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.066901] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.066904] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.066906] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:20.066909] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:20.071484] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.071493] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.071498] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:20.071504] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.071508] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.075207] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 0.93, rt: 101.00) [2024-06-25 17:00:20.075217] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SYS](tenant_id=500, io_status=sys_group_name: INSPECT_BAD_BLOCK_IO, mode: read, size: 2097152.00, iops: 0.93, rt: 768.00) [2024-06-25 17:00:20.075223] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:20.075290] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=20] [IO STATUS SYS](tenant_id=1004, io_status=sys_group_name: SLOG_IO, mode: write, size: 4915.20, iops: 4.66, rt: 76.00) [2024-06-25 17:00:20.075296] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS CONFIG](tenant_id=1004, ref_cnt=2, io_config={group_num_:0, memory_limit_:724775731200, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:100}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=12763552, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:20.075305] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075312] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075315] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075319] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075323] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075326] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075329] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075334] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075337] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075340] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075343] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075347] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075350] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075353] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075357] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.075360] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:20.077838] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62237][pnio1][T1001][YB420ABF4533-00061661718F653C-0-0] [lt=8][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:20.085736] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.085779] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=8863488, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=170, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:20.087158] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=4558183, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=198, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:20.088684] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [100465][T1003_TenantWea][T1003][Y0-0000000000000000-0-0] [lt=10] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1003, server_version={version:{val:1719306015462183000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0}, server_version_delta=4626500, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="10.191.69.52:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=58369728, succ_cluster_heartbeat_count=58369728, cluster_heartbeat_interval=50000, local_cluster_version={val:1719305992926967000, v:0}, local_cluster_delta=27161716, force_self_check=false, weak_read_refresh_interval=100000) [2024-06-25 17:00:20.100181] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=6] ====== check clog disk timer task ====== [2024-06-25 17:00:20.100195] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=7] get_disk_usage(ret=0, capacity(MB):=2073600, used(MB):=1658818) [2024-06-25 17:00:20.100209] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=6] cannot_recycle_log_size statistics(cannot_recycle_log_size=524426637471, threshold=652298158080, need_update_checkpoint_scn=false) [2024-06-25 17:00:20.108800] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=12] new mysql sessid created(conn.sessid_=3221540615, support_ssl=false) [2024-06-25 17:00:20.108824] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=10] sm conn init succ(conn.sessid_=3221540615, sess.client_addr_="10.191.69.51:48016") [2024-06-25 17:00:20.108833] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=6] accept one succ(*s={this:0x2b3add9f8030, session_id:3221540615, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:20.109047] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x2b3add9f8030, session_id:3221540615, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:20.109057] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=8] kill and revert session(conn.sessid_=3221540615, proxy_sessid=0, server_id=1, ret=0) [2024-06-25 17:00:20.109061] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:918) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=4] sock ref clean, do destroy(*s={this:0x2b3add9f8030, session_id:3221540615, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:20.109069] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:223) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=7] mark session id unused(conn.sessid_=3221540615) [2024-06-25 17:00:20.109072] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [62494][sql_nio15][T0][Y0-0000000000000000-0-0] [lt=2] connection close(sessid=3221540615, proxy_sessid=0, tenant_id=0, server_id=1, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=false, ret=0, trace_id=Y0-0000000000000000-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:0], disconnect_state=0) [2024-06-25 17:00:20.110228] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=4] re_register_parent reason(palf_id=1008, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306016008017, last_first_register_time_us=1713332561478106, register_time_us=1719306016008017, curr_time_us=1719306020110228) [2024-06-25 17:00:20.110257] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=15] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882", reason=1, leader="10.186.69.176:2882", register_time_us=1719306020110244, last_submit_register_req_time_us=1719306020110244, register_parent_reason=1) [2024-06-25 17:00:20.110265] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=7] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882") [2024-06-25 17:00:20.110270] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=4] re_register_parent reason(palf_id=1007, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306016008049, last_first_register_time_us=1713332560477990, register_time_us=1719306016008049, curr_time_us=1719306020110270) [2024-06-25 17:00:20.110282] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=6] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1007, self="10.191.69.51:2882", reason=1, leader="10.186.69.171:2882", register_time_us=1719306020110277, last_submit_register_req_time_us=1719306020110277, register_parent_reason=1) [2024-06-25 17:00:20.110287] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=5] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1007, self="10.191.69.51:2882") [2024-06-25 17:00:20.119863] INFO [SQL.JO] compute_table_location (ob_join_order.cpp:278) [62918][T1_L0_G0][T1][YB420ABF4533-000616611FBF57A2-0-0] [lt=7] succeed to calculate base table sharding info(table_id=18446744073709551614, ref_table_id=104, is_global_index=false) [2024-06-25 17:00:20.119887] INFO [SQL.JO] init_est_sel_info_for_access_path (ob_join_order.cpp:12189) [62918][T1_L0_G0][T1][YB420ABF4533-000616611FBF57A2-0-0] [lt=12] total rowcount, use statistics(table_meta_info_.table_row_count_=10, table_meta_info_.average_row_size_=2.060000000000000000e+02, table_meta_info_.micro_block_count_=1, table_meta_info_.part_size_=2.060000000000000000e+03) [2024-06-25 17:00:20.124325] WDIAG [STORAGE.TRANS] process_ (ob_gti_rpc.h:178) [62245][pnio1][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=11][errcode=0] gti rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:20.124336] WDIAG [SQL.PC] common_free (ob_lib_cache_object_manager.cpp:141) [62918][T1_L0_G0][T1][YB420ABF4533-000616611FBF57A2-0-0] [lt=11][errcode=0] set logical del time(cache_obj->get_logical_del_time()=6134803531101, cache_obj->added_lc()=false, cache_obj->get_object_id()=1202421, cache_obj->get_tenant_id()=1, lbt()="0x12a0688c 0xaa7c9e4 0x4e44d8a 0x4e36df8 0x4e2eb67 0x4e26201 0x4e23b17 0x4e13175 0x9f683b4 0x12d2c90c 0x2b3993985ea5 0x2b3993c9896d") [2024-06-25 17:00:20.125715] WDIAG get_refactored (ob_hashtable.h:1100) [62774][T1_L0_G0][T1][YB420ABF4533-0006166163BEF587-0-0] [lt=11][errcode=-4201] hashtable not init [2024-06-25 17:00:20.125722] WDIAG [SQL.SESSION] get_cursor (ob_sql_session_info.cpp:1439) [62774][T1_L0_G0][T1][YB420ABF4533-0006166163BEF587-0-0] [lt=7][errcode=-4016] get cursor info failed(cursor_id=1194334, get_sessid()=3221488526) [2024-06-25 17:00:20.125744] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB06-0-0] [lt=10] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:20.125976] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC8C5-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:20.126796] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F01D-0-0] [lt=11] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=90, avg_log_submit_to_slide_time=835) [2024-06-25 17:00:20.127434] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=63000, remain_us=24599741, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=63, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:20.129742] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [100474][T1003_ArcSender][T1003][YB420ABF4533-0006164249FE6725-0-0] [lt=9][errcode=-4018] try free send task failed(ret=-4018) [2024-06-25 17:00:20.129767] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1479027, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=218, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:20.130300] INFO eloop_run (eloop.c:143) [62232][pnio1][T0][Y0-0000000000000000-0-0] [lt=7] PNIO [ratelimit] time: 1719306020130299, bytes: 29137866649075, bw: 0.669858 MB/s, add_ts: 1000513, add_bytes: 702757 [2024-06-25 17:00:20.130313] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.130351] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.130355] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.130358] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.130360] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.130547] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325962AEF-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301546, send_timestamp=1719305997829000) [2024-06-25 17:00:20.131769] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=13603054, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=139, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:20.133038] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.133345] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4383, dropped:947, tid:62272}]) [2024-06-25 17:00:20.133459] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF59EF-0-0] [lt=9] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b61c69470, extra_info="") [2024-06-25 17:00:20.133465] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=1][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.133474] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:20.133478] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb21722e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020079578, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.133502] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=24][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb21722e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020079578, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.133514] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:20.133520] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:20.133525] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:20.133529] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:20.133532] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.133537] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:20.133543] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=222, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.133540] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=67, this=0x2b3b61c733b0, extra_info="palf_id:1006") [2024-06-25 17:00:20.133550] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=6] will sleep(sleep_us=100000, remain_us=842340, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=222, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:20.133566] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=15] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3b61c735e0, extra_info="palf_id:1006") [2024-06-25 17:00:20.133735] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AF1-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b61c696a0, extra_info="palf_id:1006") [2024-06-25 17:00:20.133740] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AF1-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=273, this=0x2b3b61c698d0, extra_info="palf_id:1006") [2024-06-25 17:00:20.134845] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=7) [2024-06-25 17:00:20.138635] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.138774] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=7][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:20.138783] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.138790] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=4][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999943, timeout_timestamp=1719306050138311) [2024-06-25 17:00:20.138797] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:20.138800] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:20.138805] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:20.138808] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=3][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:20.138812] WDIAG [SQL] open (ob_result_set.cpp:157) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=4][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:20.138815] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=3][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:20.138818] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=2][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}) [2024-06-25 17:00:20.138824] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=5][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.138829] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=4][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:20.138838] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=7][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.138840] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=2][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.138843] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:20.138850] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=4][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:20.138856] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63434][T1001_TntShared][T1001][YB420ABF4533-00061661708E6E67-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}, aret=-4013, ret=-4013) [2024-06-25 17:00:20.138859] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000) [2024-06-25 17:00:20.138865] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:20.138868] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:20.138873] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] query failed(ret=-4013, conn=0x2b3b5a082050, start=1719306020138310, sql=SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000) [2024-06-25 17:00:20.138878] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:20.138882] WDIAG [SERVER.OMT] fetch_all_srs (ob_tenant_srs.cpp:456) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute sql failed(sql=SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000, ret=-4013, tenant_id=1001) [2024-06-25 17:00:20.138908] WDIAG [SERVER.OMT] refresh_srs (ob_tenant_srs.cpp:254) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] failed to fetch ObSrsCacheSnapShot(ret=-4013, is_sys=true) [2024-06-25 17:00:20.138913] WDIAG [SERVER.OMT] runTimerTask (ob_tenant_srs.cpp:387) [63434][T1001_TntShared][T1001][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] failed to refresh sys srs(ret=-4013, tenant_srs_->remote_sys_srs_version_=0, tenant_srs_->local_sys_srs_version_=0) [2024-06-25 17:00:20.141691] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62420][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142244] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62425][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142460] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62416][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142524] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62428][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142589] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62429][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142648] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62424][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142668] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62417][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142733] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62431][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142733] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62421][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142759] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62427][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142785] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62423][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142798] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62422][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142884] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62415][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.142898] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62419][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.143035] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62418][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.143267] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62426][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.143267] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62430][RpcIO][T0][Y0-0000000000000000-0-0] [lt=5] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.143538] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62432][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.143926] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100575][T1003_L0_G0][T1003][YB420ABF4538-000616694DDF523C-0-0] [lt=12] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1006}}) [2024-06-25 17:00:20.143944] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100575][T1003_L0_G0][T1004][YB420ABF4538-000616694DDF523C-0-0] [lt=11] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306019870754003, v:0}, arg={tenant_id:1004, ls_id:{id:1006}}, result={tenant_id:1004, ls_id:{id:1006}, cur_readable_scn:{val:1719306019870754003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:20.144212] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.144220] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:20.144224] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e6ad360, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020090069, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.144249] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=24][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e6ad360, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020090069, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.144261] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:20.144266] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:20.144272] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=5][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:20.144275] WDIAG [SQL] open (ob_result_set.cpp:157) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:20.144277] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.144282] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:20.144285] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=211, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.144293] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=7] will sleep(sleep_us=100000, remain_us=2560688, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=211, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:20.149608] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8100FF693-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1, self="10.191.69.51:2882", committed size=3305) [2024-06-25 17:00:20.149926] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF7914-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86d1d470, extra_info="") [2024-06-25 17:00:20.149936] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF7914-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:20.150018] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=16] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=77, this=0x2b3a86d273b0, extra_info="palf_id:1007") [2024-06-25 17:00:20.150044] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a86d275e0, extra_info="palf_id:1007") [2024-06-25 17:00:20.150217] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F01E-0-0] [lt=6] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a86d1d6a0, extra_info="palf_id:1007") [2024-06-25 17:00:20.150222] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F01E-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=288, this=0x2b3a86d1d8d0, extra_info="palf_id:1007") [2024-06-25 17:00:20.150432] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A08B-0-0] [lt=10] [PALF STAT LOG TASK TIME](palf_id=1, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=96, avg_log_submit_to_slide_time=827) [2024-06-25 17:00:20.150440] INFO [PALF] read (log_cache.cpp:100) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=7] [PALF STAT HOT CACHE HIT RATE](palf_id=1, read_size=3722, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:20.150520] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62748][T1_ReplaySrv5][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:20.152937] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.152946] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.152951] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:20.152958] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.152963] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.155296] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABEA4-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:20.155829] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.155837] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.155842] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:20.155847] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.155852] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.159639] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.159647] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:20.159651] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:20.159657] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.159661] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.160597] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-0006164325C300FB-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301557, send_timestamp=1719305997859039) [2024-06-25 17:00:20.160880] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABF4533-00061661728FC903-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301548, send_timestamp=1719305997859331) [2024-06-25 17:00:20.162201] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=6] re_register_parent reason(palf_id=1, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306016160089, last_first_register_time_us=1713323567951674, register_time_us=1719306016160089, curr_time_us=1719306020162200) [2024-06-25 17:00:20.162222] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=10] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1, self="10.191.69.51:2882", reason=3, leader="10.186.69.171:2882", register_time_us=1719306020162212, last_submit_register_req_time_us=1719306020162212, register_parent_reason=3) [2024-06-25 17:00:20.162234] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=7] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1, self="10.191.69.51:2882") [2024-06-25 17:00:20.162240] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=4] [PALF_DUMP](palf_id=1, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:75611551, max_lsn:35325766224, max_scn:{val:1719306020148394000, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:35325766224}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:35283823184}}, last_submit_lsn:{lsn:35325765685}, last_submit_end_lsn:{lsn:35325766224}, last_submit_log_id:75611551, last_submit_log_pid:1, max_flushed_lsn:{lsn:35325765685}, max_flushed_end_lsn:{lsn:35325766224}, max_flushed_log_pid:1, committed_end_lsn:{lsn:35325766224}, last_slide_log_id:75611551, last_slide_scn:{val:1719306020148394000, v:0}, last_slide_lsn:{lsn:35325765685}, last_slide_end_lsn:{lsn:35325766224}, last_slide_log_pid:1, last_slide_log_accum_checksum:1681321914, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2b3a5d1da530}, [StateMgr]={this:0x2b3a5d1de2b0, self:"10.191.69.51:2882", palf_id:1, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, leader:"10.186.69.171:2882", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:75611552, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x2b3a5d1e4b70, allow_vote:true}, [ConfigMgr]={palf_id:1, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:1, config_seq:2}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1719306020162212, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1719306020162212, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2b3a5d1dbc70}, [ModeMgr]={palf_id:1, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1, v:0}}, accepted_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1, v:0}}, last_submit_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1, is_inited:true, min_block_max_scn:{val:1714849745850408001, v:0}, min_block_id:123, min_block_min_scn:{val:1714838998998241001, v:0}, base_lsn_for_block_gc:{lsn:35095793664}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, log_config_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, log_snapshot_meta:{version:1, base_lsn:{lsn:35095793664}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1, v:0}}}, log_meta_storage:{log_tail:{lsn:319488}, readable_log_tail:{lsn:319488}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:35364212736}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1}, block_mgr:{log_dir:"/data/clog/tenant_1/1/meta", dir_fd:581, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:66785280, block_header_serialize_buf_:0x2b3a5d1e0438, flashback_version:0}, log_storage:{log_tail:{lsn:35325766224}, readable_log_tail:{lsn:35325766224}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:35297107968}, min_scn:{val:1719300754029280001, v:0}, curr_block_id:526, palf_id:1}, block_mgr:{log_dir:"/data/clog/tenant_1/1/log", dir_fd:582, min_block_id:123, max_block_id:527, curr_writable_block_id:526}, logical_block_size_:67104768, curr_block_writable_size_:38446512, block_header_serialize_buf_:0x2b3a5d1e20b8, flashback_version:0}, palf_epoch:1, last_purge_throttling_ts:-1, this:0x2b3a5d1de9f0}, [Reconfirm]={palf_id:1, self:"10.191.69.51:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2b3a5d1de430}) [2024-06-25 17:00:20.162351] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=109][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:20.162357] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=6] [PALF_DUMP](palf_id=1, self="10.191.69.51:2882", ack_info_list=[]) [2024-06-25 17:00:20.162363] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=5][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:20.162368] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=4] current log_task status(palf_id=1, self="10.191.69.51:2882", log_id=75611552, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[]) [2024-06-25 17:00:20.164548] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:20.164557] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:20.164561] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:20.164565] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:20.164570] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:20.164573] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:20.164576] WDIAG [SQL.ENG] inner_get_next_batch_for_tsc (ob_table_scan_op.cpp:2015) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:20.164581] WDIAG [SQL.ENG] inner_get_next_batch (ob_table_scan_op.cpp:2000) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:20.164584] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4383] get next batch failed(ret=-4383, eval_ctx={batch_idx:0, batch_size:16, max_batch_size:16, frames_:0x2b813e413580}, id=2, op_name="PHY_TABLE_SCAN") [2024-06-25 17:00:20.164593] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1535) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4383] get next batch failed(ret=-4383) [2024-06-25 17:00:20.164596] WDIAG [SQL.ENG] get_next_row_vectorizely (ob_operator.cpp:1463) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4383] Failed to get_next_row vectorizedly(ret=-4383) [2024-06-25 17:00:20.164601] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_transmit_op.cpp:27) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4383] get next row from child failed(ret=-4383) [2024-06-25 17:00:20.164604] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4383] inner get next row failed(ret=-4383, type=14, op="PHY_DIRECT_TRANSMIT", op_id=1) [2024-06-25 17:00:20.164610] WDIAG [SQL.EXE] sync_send_result (ob_remote_executor_processor.cpp:283) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:20.164613] WDIAG [SQL.EXE] execute_remote_plan (ob_remote_executor_processor.cpp:469) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4383] sync send result failed(ret=-4383) [2024-06-25 17:00:20.164619] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e696fe0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221543329, assoc_session_id:3221543329, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306019898875, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2bb3c49201f8, id:3221543329, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e696fe0}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:20.164657] WDIAG [SQL.EXE] process (ob_remote_executor_processor.cpp:969) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=34][errcode=-4383] execute remote plan failed(ret=-4383, local_tenant_schema_version=1718209168713032, exec_ctx_.get_task_exec_ctx().get_query_tenant_begin_schema_version()=1718209168713032) [2024-06-25 17:00:20.165462] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.165470] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:20.165475] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:20.165481] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.165486] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.165614] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.165625] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:20.165630] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6791c060, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020111101, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.165649] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=19][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6791c060, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020111101, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.165661] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:20.165668] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:20.165672] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:20.165676] WDIAG [SQL] open (ob_result_set.cpp:157) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:20.165678] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.165682] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:20.165687] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=86, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.165694] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=6] will sleep(sleep_us=86000, remain_us=21659285, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=86, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:20.166945] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.166952] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.166955] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.166957] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:20.166959] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:20.167033] INFO [LIB] stat (utility.h:1214) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=17, stat_interval=1000000, avg (count/cost)=6, this=0x2b3a8ce1cc70, extra_info="") [2024-06-25 17:00:20.172351] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=0][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:20.172361] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:20.172911] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:20.172919] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:20.172925] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=4][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=1999812, timeout_timestamp=1719306022172371) [2024-06-25 17:00:20.172932] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:20.172935] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:20.172939] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=3][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:20.172943] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=3][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:20.172946] WDIAG [SQL] open (ob_result_set.cpp:157) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=2][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:20.172948] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:20.172952] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=4][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}) [2024-06-25 17:00:20.172957] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=4][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.172963] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=5][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:20.172969] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=5][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.172973] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=4][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.172975] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:20.172981] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3A-0-0] [lt=3][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:20.172986] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63182][T1002_TenantInf][T1002][YB420ABF4533-000616616F6E9D3A-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, aret=-4013, ret=-4013) [2024-06-25 17:00:20.172990] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:20.172995] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:20.172998] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:20.173002] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] query failed(ret=-4013, conn=0x2b3a78d44eb0, start=1719306020172508, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:20.173007] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:20.173010] WDIAG [SHARE] load_pure_tenant_info_ (ob_tenant_info_proxy.cpp:413) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] failed to read(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", exec_tenant_id=1001, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:20.173039] WDIAG [SHARE] load_tenant_info (ob_tenant_info_proxy.cpp:368) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4013] failed to load purge tenant info(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, for_update=false) [2024-06-25 17:00:20.174071] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63309][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=11] start do ls ha handler(ls_id_array_=[{id:1010}, {id:1008}, {id:1009}, {id:1006}, {id:1007}]) [2024-06-25 17:00:20.175306] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.175316] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:20.175320] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:20.175326] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.175331] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.177882] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62240][pnio1][T1001][YB420ABF4533-00061661718F653D-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.180285] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [100368][T1003_Occam][T1003][Y0-0000000000000000-0-0] [lt=10] ====== tenant freeze timer task ====== [2024-06-25 17:00:20.180299] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [100368][T1003_Occam][T1003][Y0-0000000000000000-0-0] [lt=7][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-06-25 17:00:20.185795] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.185845] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=10][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:170, local_retry_times:170, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:20.185855] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.185858] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.185861] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:20.185868] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:20.187225] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=10][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:198, local_retry_times:198, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:20.187240] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=13][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.187244] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.187247] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:20.187254] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:20.190506] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=10][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:63, local_retry_times:63, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:20.190518] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=11][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.190523] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.190526] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:20.190532] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:20.190582] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=12] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289091137536, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:20.193116] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.201381] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.201392] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.201399] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:20.201403] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:20.201408] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:20.201411] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:20.201415] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:20.201417] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:20.201420] WDIAG [SQL.ENG] inner_get_next_batch_for_tsc (ob_table_scan_op.cpp:2015) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:20.201422] WDIAG [SQL.ENG] inner_get_next_batch (ob_table_scan_op.cpp:2000) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:20.201425] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] get next batch failed(ret=-4383, eval_ctx={batch_idx:0, batch_size:16, max_batch_size:16, frames_:0x2bc018812e10}, id=2, op_name="PHY_TABLE_SCAN") [2024-06-25 17:00:20.201432] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1535) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4383] get next batch failed(ret=-4383) [2024-06-25 17:00:20.201434] WDIAG [SQL.ENG] get_next_row_vectorizely (ob_operator.cpp:1463) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] Failed to get_next_row vectorizedly(ret=-4383) [2024-06-25 17:00:20.201437] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_transmit_op.cpp:27) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] get next row from child failed(ret=-4383) [2024-06-25 17:00:20.201440] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] inner get next row failed(ret=-4383, type=14, op="PHY_DIRECT_TRANSMIT", op_id=1) [2024-06-25 17:00:20.201445] WDIAG [SQL.EXE] sync_send_result (ob_remote_executor_processor.cpp:283) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:20.201447] WDIAG [SQL.EXE] execute_remote_plan (ob_remote_executor_processor.cpp:469) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4383] sync send result failed(ret=-4383) [2024-06-25 17:00:20.201451] WDIAG [SQL.EXE] process (ob_remote_executor_processor.cpp:969) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] execute remote plan failed(ret=-4383, local_tenant_schema_version=1718209168713032, exec_ctx_.get_task_exec_ctx().get_query_tenant_begin_schema_version()=1718209168713032) [2024-06-25 17:00:20.218106] INFO [STORAGE.BLKMGR] inspect_bad_block (ob_block_manager.cpp:1712) [62325][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=13] check macro block(block_info={ref_cnt:1, is_free:false, access_time:1717780759553433}, time_interval=1525260664671) [2024-06-25 17:00:20.225737] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8013-0-0] [lt=8] push gts task success(*task={:""}) [2024-06-25 17:00:20.226009] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC8C9-0-0] [lt=3] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86de7470, extra_info="") [2024-06-25 17:00:20.226098] INFO [LIB] stat (utility.h:1197) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=69, this=0x2b3a86df13b0, extra_info="palf_id:1008") [2024-06-25 17:00:20.226111] INFO [LIB] stat (utility.h:1214) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a8ce1daf0, extra_info="") [2024-06-25 17:00:20.226129] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=9, this=0x2b3a86df15e0, extra_info="palf_id:1008") [2024-06-25 17:00:20.226830] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.226857] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F021-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a86de76a0, extra_info="palf_id:1008") [2024-06-25 17:00:20.226862] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F021-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=837, this=0x2b3a86de78d0, extra_info="palf_id:1008") [2024-06-25 17:00:20.227790] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8013-0-0] [lt=9] rpc statistics(total_trans_msg_count=2, total_batch_msg_count=0) [2024-06-25 17:00:20.228243] INFO [STORAGE.TRANS] process (ob_trans_rpc.cpp:142) [100566][T1003_L0_G0][T1003][YB420ABF4534-0006164516C9775C-0-0] [lt=10] transaction rpc statistics(total_rt=112, total_process=16, avg_rt=6) [2024-06-25 17:00:20.229841] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:218, local_retry_times:218, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:20.229851] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.229855] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.229858] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:20.229862] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:20.230184] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-0006164323C50517-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5386, fly_ts=22301552, send_timestamp=1719305997928632) [2024-06-25 17:00:20.230366] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.230402] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.230407] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.230410] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.230415] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.230439] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:20.230597] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-0006164325962AF4-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301546, send_timestamp=1719305997929050) [2024-06-25 17:00:20.230634] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164325962AF5-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301548, send_timestamp=1719305997929085) [2024-06-25 17:00:20.231845] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=8][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:139, local_retry_times:139, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:20.231855] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.231858] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.231862] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:20.231867] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:20.232464] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8042FDE53-0-0] [lt=3] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1006, child={server:"10.186.69.171:2882", region:"sys_region", register_time_us:1719306020231596, keepalive_ts:-1}) [2024-06-25 17:00:20.232473] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8042FDE53-0-0] [lt=9] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1006, self="10.191.69.51:2882", child={server:"10.186.69.171:2882", region:"sys_region", register_time_us:1719306020231596, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:20.233092] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.233613] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=7][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:222, local_retry_times:222, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:20.233625] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.233629] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.233633] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:20.233639] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:20.235082] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=1][errcode=-4383] [2024-06-25 17:00:20.235087] WDIAG [SQL.EXE] setup_next_scanner (ob_direct_receive_op.cpp:207) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=5][errcode=-4383] error occurring in the remote sql execution, please use the current TRACE_ID to grep the original error message on the remote_addr.(ret=-4383, remote_addr="10.191.69.52:2882") [2024-06-25 17:00:20.235092] WDIAG [SQL.EXE] inner_open (ob_direct_receive_op.cpp:53) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=5][errcode=-4383] failed to setup first scanner(ret=-4383) [2024-06-25 17:00:20.235095] WDIAG [SQL.ENG] open (ob_operator.cpp:714) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=3][errcode=-4383] Open this operator failed(ret=-4383, op_type="PHY_DIRECT_RECEIVE") [2024-06-25 17:00:20.235100] WDIAG [SQL.EXE] open (ob_execute_result.cpp:104) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=4][errcode=-4383] open operator failed(ret=-4383) [2024-06-25 17:00:20.235103] WDIAG [SQL] open_result (ob_result_set.cpp:210) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=2][errcode=-4383] fail open main query(ret=-4383) [2024-06-25 17:00:20.235106] WDIAG [SQL] open (ob_result_set.cpp:159) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.235108] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.235112] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}) [2024-06-25 17:00:20.235116] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1002, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, retry_cnt=33, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1718209168713032) [2024-06-25 17:00:20.235122] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=5] will sleep(sleep_us=33000, remain_us=524075609, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=33, v.err_=-4383, timeout_timestamp=1719306544310730) [2024-06-25 17:00:20.238689] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.239210] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164319BD3E67-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5407, fly_ts=22301545, send_timestamp=1719305997937664) [2024-06-25 17:00:20.239560] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.239571] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:20.239575] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e7702c0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020185803, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.239597] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=22][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e7702c0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020185803, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.239610] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:20.239617] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:20.239622] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:20.239625] WDIAG [SQL] open (ob_result_set.cpp:157) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:20.239628] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.239625] INFO eloop_run (eloop.c:143) [62250][pnio2][T0][Y0-0000000000000000-0-0] [lt=11] PNIO [ratelimit] time: 1719306020239625, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007609, add_bytes: 0 [2024-06-25 17:00:20.239632] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:20.239636] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=171, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.239644] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=6] will sleep(sleep_us=100000, remain_us=8709624, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=171, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:20.240873] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.240879] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=6][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:20.240882] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6ab67940, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020186859, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.240899] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6ab67940, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020186859, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.240910] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999998, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:20.240916] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:20.240920] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:20.240923] WDIAG [SQL] open (ob_result_set.cpp:157) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:20.240926] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.240931] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}) [2024-06-25 17:00:20.240934] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, retry_cnt=199, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.240940] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=4404400, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=199, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:20.244354] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:211, local_retry_times:211, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:20.244362] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=7][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.244365] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.244368] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:20.244369] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.244374] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:20.244379] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:20.244383] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6d6d5820, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020189979, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.244400] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6d6d5820, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020189979, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.244411] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:20.244417] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:20.244421] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:20.244423] WDIAG [SQL] open (ob_result_set.cpp:157) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:20.244427] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.244430] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:20.244435] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=64, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.244440] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5] will sleep(sleep_us=64000, remain_us=24482735, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=64, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:20.244961] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [99399][T1004_LSArchive][T1004][YB420ABF4533-00061642492E6725-0-0] [lt=9] gc stale ls task succ [2024-06-25 17:00:20.251757] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:86, local_retry_times:86, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:20.251767] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.251770] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.251772] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:20.251779] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:20.255418] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=49705027, this={dio_aligned_buf:{buf_write_offset:189, buf_padding_size:3907, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:876850988, truncate_used_ts:118323694}, log_block_size:67108864, dir_fd:861, io_fd:849}, aligned_buf_len=4096, aligned_buf="LH", aligned_block_offset=49704960, buf_len=122, total_write_size_=12294, total_write_size_after_dio_=425984, ob_pwrite_used_ts=7243, count=101, each_pwrite_cost=71) [2024-06-25 17:00:20.260618] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325C300FF-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301546, send_timestamp=1719305997959072) [2024-06-25 17:00:20.260919] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABF4538-00061669548FCC21-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301548, send_timestamp=1719305997959370) [2024-06-25 17:00:20.261178] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [100466][T1003_TxCkpt][T1003][Y0-0000000000000000-0-0] [lt=6] ====== checkpoint timer task ====== [2024-06-25 17:00:20.261188] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:193) [100466][T1003_TxCkpt][T1003][Y0-0000000000000000-0-0] [lt=6] no logstream(ret=0, ls_cnt=0) [2024-06-25 17:00:20.267002] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.267009] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.267011] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.267014] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:20.267018] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:20.267215] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-000616432547F024-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301541, send_timestamp=1719305997965673) [2024-06-25 17:00:20.268185] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:33, local_retry_times:33, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:20.268195] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.268200] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.268202] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:20.268209] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:20.278841] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301559, send_timestamp=1719305997977282) [2024-06-25 17:00:20.279469] WDIAG [RPC] send (ob_poc_rpc_proxy.h:156) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4012] sync rpc execute fail(ret=-4012, addr="10.191.69.52:2882", pcode=1315) [2024-06-25 17:00:20.279481] WDIAG [SQL.ENG] query_statistics (ob_px_tenant_target_monitor.cpp:275) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=8][errcode=-4012] send rpc to query statistics failed, need send refresh all(ret=-4012) [2024-06-25 17:00:20.279487] WDIAG [SQL.ENG] refresh_statistics (ob_px_tenant_target_monitor.cpp:114) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] query statistics failed(ret=-4012) [2024-06-25 17:00:20.279491] WDIAG [COMMON] operator() (ob_px_target_mgr.h:158) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] target monitor refresh statistics failed(ret=-4012, px_tenant_info={v:1002}, px_res_info->get_target_monitor()={is_init:true, tenant_id:1002, server:"10.191.69.51:2882", dummy_cache_leader:"10.191.69.52:2882", role:2}) [2024-06-25 17:00:20.279638] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.279651] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.282568] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.283594] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.283605] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:20.283609] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78dcefe0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020229763, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.283627] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78dcefe0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020229763, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.283638] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=10][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:20.283644] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:20.283647] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:20.283651] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:20.283653] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.283656] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:20.283661] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=219, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.283667] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=1325127, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=219, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:20.285642] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.285651] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:20.285654] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a693555c0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020231870, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.285669] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=15][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.285685] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=13449139, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=140, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:20.285852] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.286477] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.287264] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.287274] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:20.287278] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb2197360, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020232923, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.287296] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=18][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb2197360, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020232923, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.287309] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:20.287315] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:20.287319] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:20.287322] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:20.287325] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.287330] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:20.287334] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=223, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.287340] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=688551, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=223, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:20.287799] INFO [SHARE] blacklist_loop_ (ob_server_blacklist.cpp:313) [62346][Blacklist][T0][Y0-0000000000000000-0-0] [lt=8] blacklist_loop exec finished(cost_time=12, is_enabled=true, send_cnt=0) [2024-06-25 17:00:20.292337] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.298081] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2406900, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=212, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:20.302115] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.305403] INFO [CLOG] run1 (ob_garbage_collector.cpp:1338) [63450][T1001_GCCollect][T1001][Y0-0000000000000000-0-0] [lt=3] Garbage Collector is running(seq_=598231, gc_interval=10000000) [2024-06-25 17:00:20.305427] INFO [CLOG] gc_check_member_list_ (ob_garbage_collector.cpp:1431) [63450][T1001_GCCollect][T1001][Y0-0000000000000000-0-0] [lt=9] gc_check_member_list_ cost time(ret=0, time_us=14) [2024-06-25 17:00:20.305434] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1703) [63450][T1001_GCCollect][T1001][Y0-0000000000000000-0-0] [lt=5] execute_gc cost time(ret=0, time_us=0) [2024-06-25 17:00:20.305442] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1703) [63450][T1001_GCCollect][T1001][Y0-0000000000000000-0-0] [lt=6] execute_gc cost time(ret=0, time_us=0) [2024-06-25 17:00:20.305446] INFO [SERVER] handle (ob_safe_destroy_handler.cpp:240) [63450][T1001_GCCollect][T1001][Y0-0000000000000000-0-0] [lt=3] ObSafeDestroyHandler start process [2024-06-25 17:00:20.305585] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.305602] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=87000, remain_us=21519377, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=87, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:20.305949] INFO [STORAGE] schedule_all_tablets_minor (ob_tenant_tablet_scheduler.cpp:511) [63480][T1001_MergeLoop][T1001][YB420ABF4533-0006164244AF29E1-0-0] [lt=4] start schedule all tablet minor merge(minor_ls_tablet_iter_={ls_idx:-1, ls_ids:[], tablet_idx:0, tablet_ids_.count():0, schedule_tablet_cnt:0, max_batch_tablet_cnt:50000, report_scn_flag:false}) [2024-06-25 17:00:20.305960] INFO [STORAGE] runTimerTask (ob_tenant_tablet_scheduler.cpp:135) [63480][T1001_MergeLoop][T1001][YB420ABF4533-0006164244AF29E1-0-0] [lt=11] MergeLoopTask(cost_ts=25) [2024-06-25 17:00:20.308505] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=6][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.315099] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.317939] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.319873] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.322819] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABF4533-0006166171EF4872-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301546, send_timestamp=1719305998021272) [2024-06-25 17:00:20.322860] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.325832] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB0E-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b74f470, extra_info="") [2024-06-25 17:00:20.325931] INFO [LIB] stat (utility.h:1197) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=81, this=0x2b3b5b7593b0, extra_info="palf_id:1007") [2024-06-25 17:00:20.325941] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:20.325950] INFO [LIB] stat (utility.h:1214) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3afcbffaf0, extra_info="") [2024-06-25 17:00:20.325963] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=11, this=0x2b3b5b7595e0, extra_info="palf_id:1007") [2024-06-25 17:00:20.326635] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AF8-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5b74f6a0, extra_info="palf_id:1007") [2024-06-25 17:00:20.326640] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AF8-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=831, this=0x2b3b5b74f8d0, extra_info="palf_id:1007") [2024-06-25 17:00:20.328667] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.330410] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.330442] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.330446] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.330449] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.330451] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.330683] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC060-0006164325962AFA-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305998029138) [2024-06-25 17:00:20.330703] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AFA-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1005, self="10.191.69.51:2882", committed size=1342) [2024-06-25 17:00:20.333149] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.333722] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962AFB-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=83, avg_log_submit_to_slide_time=271) [2024-06-25 17:00:20.333728] INFO [PALF] read (log_cache.cpp:100) [99521][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1006, read_size=1342, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:20.334911] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=10] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=6) [2024-06-25 17:00:20.338478] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.338743] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.341000] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=6][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.342808] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABF4533-0006166172BEBEA4-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301546, send_timestamp=1719305998041261) [2024-06-25 17:00:20.343621] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63467][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=1, size_used=7213, mem_used=12478464) [2024-06-25 17:00:20.350588] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=11] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:20.353691] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.356340] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.362302] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=65000, remain_us=24364873, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=65, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:20.362419] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62458][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.362770] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B61-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf09d470, extra_info="") [2024-06-25 17:00:20.362841] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=77, this=0x2b3aaf0a73b0, extra_info="palf_id:1009") [2024-06-25 17:00:20.362867] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=7, this=0x2b3aaf0a75e0, extra_info="palf_id:1009") [2024-06-25 17:00:20.362890] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62455][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.363210] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62460][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.363242] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62459][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.363366] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62456][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.363523] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62457][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.363652] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F027-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf09d6a0, extra_info="palf_id:1009") [2024-06-25 17:00:20.363657] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F027-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=842, this=0x2b3aaf09d8d0, extra_info="palf_id:1009") [2024-06-25 17:00:20.363667] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62454][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.364771] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [62725][T1_LSArchiveMgr][T1][YB420ABF4533-000616423A1E6725-0-0] [lt=10] gc stale ls task succ [2024-06-25 17:00:20.366620] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62466][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.366655] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62464][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.366904] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62469][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367044] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62461][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367052] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.367056] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62443][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367057] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.367060] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.367063] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:20.367065] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:20.367109] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62441][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367113] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62463][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367112] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62434][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367219] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62442][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367245] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62465][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367246] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62468][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367256] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62446][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367355] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62451][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367438] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62444][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367466] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62470][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367578] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62471][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367639] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62445][BatchIO][T0][Y0-0000000000000000-0-0] [lt=9] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367673] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62452][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367678] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62449][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367703] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62437][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367773] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62440][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367801] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62448][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367802] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62438][BatchIO][T0][Y0-0000000000000000-0-0] [lt=10] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.367896] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62462][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=4] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.368045] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62467][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.368165] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62436][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.368172] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62450][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.368248] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62473][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.368263] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62439][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.368376] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62435][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.368530] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62447][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.373388] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62475][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:20.377618] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [63468][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=6] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1001, server_version={version:{val:1718277118520766000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0}, server_version_delta=1028901856851, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="10.191.69.52:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=58469791, succ_cluster_heartbeat_count=48409026, cluster_heartbeat_interval=50000, local_cluster_version={val:1718276655951614000, v:0}, local_cluster_delta=1029364426003, force_self_check=false, weak_read_refresh_interval=100000) [2024-06-25 17:00:20.377651] INFO [STORAGE.TRANS] generate_new_version (ob_tenant_weak_read_server_version_mgr.cpp:120) [63468][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=19] [WRS] update tenant weak read server version(tenant_id=1001, server_version={version:{val:1718277118520766000, v:0}, total_part_count:0, valid_inner_part_count:0, valid_user_part_count:0, epoch_tstamp:1719306020377647}, version_delta=1028901856885) [2024-06-25 17:00:20.378157] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100131][T1004_L0_G0][T1004][YB420ABF4534-00061645070C78FF-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.380524] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [62735][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=7] start do ls ha handler(ls_id_array_=[{id:1}]) [2024-06-25 17:00:20.382227] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100201][T1004_L0_G0][T1004][YB420ABF4534-00061645070C78FF-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.385908] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.387400] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:223, local_retry_times:223, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:20.387412] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.387415] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.387418] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:20.387424] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:20.390272] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.390369] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [63254][T1002_ReqMemEvi][T1002][YB420ABF4533-0006165F60BFEA4C-0-0] [lt=0][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.390381] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63254][T1002_ReqMemEvi][T1002][YB420ABF4533-0006165F60BFEA4C-0-0] [lt=0] will sleep(sleep_us=8000, remain_us=11258878, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=8, v.err_=-4383, timeout_timestamp=1719306031649258) [2024-06-25 17:00:20.390807] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=14] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289111621632, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:20.393265] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=8556002, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=172, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:20.394610] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=4250730, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=200, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:20.398145] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=9][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.419598] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.424872] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325210E41-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=330, fly_ts=22301547, send_timestamp=1719305998123324) [2024-06-25 17:00:20.426773] INFO [LIB] stat (utility.h:1197) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=10] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=7, this=0x2b3ae6e6c8f8, extra_info="") [2024-06-25 17:00:20.426782] INFO [LIB] stat (utility.h:1197) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=1, this=0x2b3ae6e6cb28, extra_info="") [2024-06-25 17:00:20.426874] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=23] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:20.430467] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.430502] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.430507] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.430510] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.430513] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.430713] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABDC060-0006164325962AFE-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301546, send_timestamp=1719305998129166) [2024-06-25 17:00:20.430728] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-0006164325962AFF-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301554, send_timestamp=1719305998129173) [2024-06-25 17:00:20.433201] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.433717] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B00-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=1342) [2024-06-25 17:00:20.437091] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABA45B0-00061644F03C36DB-0-0] [lt=7] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1008}}) [2024-06-25 17:00:20.437108] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABA45B0-00061644F03C36DB-0-0] [lt=10] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306020138952002, v:0}, arg={tenant_id:1004, ls_id:{id:1008}}, result={tenant_id:1004, ls_id:{id:1008}, cur_readable_scn:{val:1719306020138952002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:20.437501] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1171293, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=220, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:20.438799] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.439528] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=13295295, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=141, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:20.441092] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.441103] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:20.441107] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb21955c0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020387426, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.441127] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=20][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb21955c0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221645591, assoc_session_id:3221645591, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306020387426, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:20.441139] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:20.441145] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:20.441148] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:20.441152] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:20.441155] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:20.441158] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:20.441164] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=224, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.441169] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=4] will sleep(sleep_us=100000, remain_us=534721, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=224, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:20.446487] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=88000, remain_us=21378492, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=88, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:20.447396] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABA45AB-000616C80A1FC7F6-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1007}}) [2024-06-25 17:00:20.447405] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABA45AB-000616C80A1FC7F6-0-0] [lt=5] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306020045218003, v:0}, arg={tenant_id:1004, ls_id:{id:1007}}, result={tenant_id:1004, ls_id:{id:1007}, cur_readable_scn:{val:1719306020045218003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:20.449927] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [63197][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=4] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1002) [2024-06-25 17:00:20.449937] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [63197][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=8] start lib cache evict(tenant_id=1002, mem_hold=47325184, mem_limit=53687090, cache_obj_num=196, cache_node_num=194) [2024-06-25 17:00:20.449943] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [63197][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=5] end lib cache evict(tenant_id=1002, cache_evict_num=0, mem_hold=47325184, mem_limit=53687090, cache_obj_num=196, cache_node_num=194) [2024-06-25 17:00:20.449949] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [63197][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=5] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:20.451750] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2253231, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=213, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:20.452073] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [63197][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=3] Dumping All Cache Objs(alloc_obj_list.count()=196, alloc_obj_list=[{obj_id:805, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192706}, {obj_id:150403, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223575}, {obj_id:154857, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:154984, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:155022, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:155068, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:155127, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:9603, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:9604, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238685}, {obj_id:9605, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:9608, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:157147, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286689}, {obj_id:9706, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:9720, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:9723, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:9725, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157845}, {obj_id:157233, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:3, added_to_lc:true, mem_used:180931}, {obj_id:157235, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176677}, {obj_id:157236, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:3, added_to_lc:true, mem_used:177270}, {obj_id:157237, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:3, added_to_lc:true, mem_used:176478}, {obj_id:157238, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:157239, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:157240, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:157241, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205876}, {obj_id:157244, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198032}, {obj_id:157247, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180301}, {obj_id:157248, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:157249, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:157250, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:166015}, {obj_id:157252, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:9804, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:157709, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:157710, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158080, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158169, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158170, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173478}, {obj_id:158171, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173774}, {obj_id:158173, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158175, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178326}, {obj_id:158181, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158190, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158191, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158192, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157664}, {obj_id:158254, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184115}, {obj_id:158256, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158257, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158262, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:336809}, {obj_id:158265, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158275, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178640}, {obj_id:158297, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158460, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:158692, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:310086}, {obj_id:158923, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303391}, {obj_id:159573, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176876}, {obj_id:159574, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:159575, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:160136, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:366914}, {obj_id:160138, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175019}, {obj_id:160141, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179388}, {obj_id:160145, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175715}, {obj_id:111287, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:111292, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:111293, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:111365, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:111437, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124490, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:3, added_to_lc:true, mem_used:190714}, {obj_id:124492, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124493, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124494, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124497, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183755}, {obj_id:124504, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176366}, {obj_id:124505, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177408}, {obj_id:124506, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:124509, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124510, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:124512, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124519, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124520, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124521, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124522, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124527, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124528, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124533, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124534, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124536, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124543, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124544, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124547, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124551, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196281}, {obj_id:124552, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196281}, {obj_id:124563, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124567, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124568, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124570, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181747}, {obj_id:124575, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:162310}, {obj_id:124581, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124583, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124588, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124591, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124604, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124619, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124631, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124632, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124637, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124643, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124681, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124690, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124693, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124700, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124708, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124720, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124764, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124788, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221466}, {obj_id:124819, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124821, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124822, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124823, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124825, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124826, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124827, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124828, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124831, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124832, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124833, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124835, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:125020, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196855}, {obj_id:125022, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:302321}, {obj_id:125030, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:125037, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216295}, {obj_id:125039, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:125044, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:125049, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:125051, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:125055, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:125120, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:284877}, {obj_id:136094, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238009}, {obj_id:136144, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180321}, {obj_id:136170, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136175, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136177, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136181, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136182, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136185, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136186, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136189, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136194, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136202, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136204, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136205, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136206, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136211, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136215, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136216, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136217, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136226, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136227, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136231, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136232, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136237, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136238, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136240, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136266, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:172302}, {obj_id:136274, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179184}, {obj_id:136282, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181485}, {obj_id:136285, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182120}, {obj_id:136287, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180509}, {obj_id:136307, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136309, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136312, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136313, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136315, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136316, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136317, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136319, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136322, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136325, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136333, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177763}, {obj_id:136335, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233739}, {obj_id:136336, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:323415}, {obj_id:136337, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136338, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136339, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136341, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136343, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136344, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136348, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136349, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136352, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136354, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136355, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232039}, {obj_id:136356, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196473}, {obj_id:136357, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136358, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136360, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:136361, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:137876, tenant_id:1002, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}]) [2024-06-25 17:00:20.452260] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [63197][T1002_PlanCache][T1002][Y0-0000000000000000-0-0] [lt=186] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:20.455452] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABEB0-0-0] [lt=8] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b6d5470, extra_info="") [2024-06-25 17:00:20.455545] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=13] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=73, this=0x2b3b5b6df3b0, extra_info="palf_id:1008") [2024-06-25 17:00:20.455571] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=9, this=0x2b3b5b6df5e0, extra_info="palf_id:1008") [2024-06-25 17:00:20.456063] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.456305] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B01-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5b6d56a0, extra_info="palf_id:1008") [2024-06-25 17:00:20.456310] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B01-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=847, this=0x2b3b5b6d58d0, extra_info="palf_id:1008") [2024-06-25 17:00:20.460734] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164325C30107-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301550, send_timestamp=1719305998159183) [2024-06-25 17:00:20.461022] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABF4538-00061669548FCC29-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301550, send_timestamp=1719305998159472) [2024-06-25 17:00:20.462250] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=9] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1, need_refresh=false, gts_local_cache={srr:[mts=1719306019961728], gts:1719306019961169000, latest_srr:[mts=1719306020462240]}) [2024-06-25 17:00:20.462262] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=11] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-06-25 17:00:20.462287] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=5] gts statistics(tenant_id=1003, gts_rpc_cnt=30, get_gts_cache_cnt=110, get_gts_with_stc_cnt=20, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-06-25 17:00:20.462400] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [63471][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=4] ====== check clog disk timer task ====== [2024-06-25 17:00:20.462411] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [63471][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=9] get_disk_usage(ret=0, capacity(MB):=614, used(MB):=0) [2024-06-25 17:00:20.462419] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [63471][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=5] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=193273528, need_update_checkpoint_scn=false) [2024-06-25 17:00:20.462664] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62237][pnio1][T0][YB420ABF4533-0006166171AF65E3-0-0] [lt=12][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:20.462675] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62237][pnio1][T0][YB420ABF4533-0006166171AF65E3-0-0] [lt=11] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:20.462688] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62237][pnio1][T0][YB420ABF4533-0006166171AF65E3-0-0] [lt=6] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306020462272]}) [2024-06-25 17:00:20.467118] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.467123] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.467126] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.467128] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:20.467130] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:20.467358] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-000616432547F02C-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301542, send_timestamp=1719305998165815) [2024-06-25 17:00:20.478082] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62233][pnio1][T1001][YB420ABF4533-00061661718F6540-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.481165] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=66000, remain_us=24246010, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=66, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:20.485964] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.486631] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [100471][T1003_LSArchive][T1003][YB420ABF4533-0006164249CE6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:20.493330] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=9][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.493530] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [100481][T1003_HAService][T1003][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:20.499766] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [99409][T1004_HAService][T1004][Y0-0000000000000000-0-0] [lt=10] start do ls ha handler(ls_id_array_=[{id:1039}, {id:1005}, {id:1008}, {id:1007}, {id:1006}]) [2024-06-25 17:00:20.520490] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.523368] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.526244] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1342) [2024-06-25 17:00:20.527164] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.530515] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.530543] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=16, this=0x2b3b5a817470, extra_info="") [2024-06-25 17:00:20.530552] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5a884e50, extra_info="id:1005") [2024-06-25 17:00:20.530566] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=14] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.530569] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.530571] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.530574] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.530630] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=75, this=0x2b3b5a8213b0, extra_info="palf_id:1005") [2024-06-25 17:00:20.530647] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8215e0, extra_info="palf_id:1005") [2024-06-25 17:00:20.530745] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325962B03-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305998229200) [2024-06-25 17:00:20.530804] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325962B04-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301545, send_timestamp=1719305998229258) [2024-06-25 17:00:20.530844] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B04-0-0] [lt=3] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8176a0, extra_info="palf_id:1005") [2024-06-25 17:00:20.530851] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B04-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=300, this=0x2b3b5a8178d0, extra_info="palf_id:1005") [2024-06-25 17:00:20.532953] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.533255] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.534977] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=9] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=5) [2024-06-25 17:00:20.535977] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [100464][T1003_ReqMemEvi][T1003][Y0-0000000000000000-0-0] [lt=8] sql audit evict task end(request_manager_->get_tenant_id()=1003, evict_high_mem_level=1932735283, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=82994, mem_used=195495936) [2024-06-25 17:00:20.537548] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.538853] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.541234] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=6][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:224, local_retry_times:224, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:20.541243] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=7][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.541246] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=2][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:20.541248] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:20.541254] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:20.542723] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.546864] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=8402403, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=173, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:20.547371] WDIAG [RPC] send (ob_poc_rpc_proxy.h:156) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=0][errcode=-4012] sync rpc execute fail(ret=-4012, addr="10.191.69.52:2882", pcode=1282) [2024-06-25 17:00:20.547387] WDIAG [SQL.EXE] task_execute (ob_executor_rpc_impl.cpp:81) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=11][errcode=-4012] rpc task_execute fail(ret=-4012, tenant_id=1002, svr="10.191.69.52:2882", timeout=1697866, timeout_timestamp=1719306020551910) [2024-06-25 17:00:20.547397] WDIAG [SQL.EXE] execute (ob_remote_task_executor.cpp:99) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=8][errcode=-4012] fail post task(ret=-4012) [2024-06-25 17:00:20.547402] WDIAG [SQL.EXE] execute (ob_remote_job_executor.cpp:55) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=4][errcode=-4012] fail execute task(ret=-4012, *task_info={task_loc:{server:"10.191.69.52:2882", ob_task_id:{ob_job_id:{ob_execution_id:{server:"10.191.69.51:2882", execution_id:18446744073709551615, task_type:0, hash:4219408603045787357}, job_id:4037275}, task_id:0, task_cnt:0}}, range_location:{part_locs:[], server:"0.0.0.0:0"}, location_idx:0, location_idx_list:[0], state:5, slice_count_pos:[], background:false, retry_times:0, location_idx_list:[0]}) [2024-06-25 17:00:20.547418] WDIAG [SQL.EXE] execute_with_plan (ob_remote_scheduler.cpp:124) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=16][errcode=-4012] fail execute remote job(ret=-4012) [2024-06-25 17:00:20.547430] WDIAG [SQL.EXE] execute_with_plan (ob_remote_scheduler.cpp:141) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=3][errcode=-4012] fail to schedule, print job's status(ret=-4012, print_ret=0, job_status="{job_count:2, {task_count:0, state statistics:{}}, {task_count:1, task_info:{loc:"10.191.69.52:2882"ctrl:"10.191.69.51:2882"eid:18446744073709551615jid:4037275tid:0pull_sid:0state:5}state statistics:{failed:1}}}") [2024-06-25 17:00:20.547439] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:57) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=5][errcode=-4012] execute with plan failed(ret=-4012) [2024-06-25 17:00:20.547444] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=4][errcode=-4012] fail execute plan(ret=-4012) [2024-06-25 17:00:20.547448] WDIAG [SQL] open (ob_result_set.cpp:157) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=4][errcode=-4012] execute plan failed(ret=-4012) [2024-06-25 17:00:20.547452] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=3][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:20.547455] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=2][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}) [2024-06-25 17:00:20.547461] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=5][errcode=-4012] execute failed(ret=-4012, tenant_id=1002, executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}, retry_cnt=14, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1718209168713032) [2024-06-25 17:00:20.547467] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=5][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:14, local_retry_times:14, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:20.547475] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=6][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:20.547477] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=2][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:20.547479] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=2][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:20.547486] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=3][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:20.547491] INFO [SERVER] process_final (ob_inner_sql_connection.cpp:734) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=4] slow inner sql(last_ret=-4012, sql={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}, process_time=29995571) [2024-06-25 17:00:20.547495] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63256][T1002_TntShared][T1002][YB420ABF4533-000616604A6F3710-0-0] [lt=4][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000"}, aret=-4012, ret=-4012) [2024-06-25 17:00:20.547501] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63256][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1002, sql=SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000) [2024-06-25 17:00:20.547505] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63256][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1002) [2024-06-25 17:00:20.547510] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63256][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1002) [2024-06-25 17:00:20.547513] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63256][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] query failed(ret=-4012, conn=0x2b3afefeabd0, start=1719305990551910, sql=SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000) [2024-06-25 17:00:20.547520] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63256][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4012] read failed(ret=-4012) [2024-06-25 17:00:20.547523] WDIAG [SERVER.OMT] fetch_all_srs (ob_tenant_srs.cpp:456) [63256][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=2][errcode=-4012] execute sql failed(sql=SELECT * FROM __all_spatial_reference_systems WHERE (SRS_ID < 70000000 AND SRS_ID != 0) OR SRS_ID > 2000000000, ret=-4012, tenant_id=1002) [2024-06-25 17:00:20.547558] WDIAG [SERVER.OMT] refresh_srs (ob_tenant_srs.cpp:254) [63256][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4012] failed to fetch ObSrsCacheSnapShot(ret=-4012, is_sys=true) [2024-06-25 17:00:20.547563] WDIAG [SERVER.OMT] runTimerTask (ob_tenant_srs.cpp:387) [63256][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] failed to refresh sys srs(ret=-4012, tenant_srs_->remote_sys_srs_version_=0, tenant_srs_->local_sys_srs_version_=0) [2024-06-25 17:00:20.548377] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=4096964, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=201, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:20.548737] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F8-0-0] [lt=8][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=1642, pos=316, bitmap=NULL) [2024-06-25 17:00:20.548920] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F8-0-0] [lt=9][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=12, len=172, pos=145, bitmap=NULL) [2024-06-25 17:00:20.549121] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48F8-0-0] [lt=5][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=333, pos=303, bitmap=NULL) [2024-06-25 17:00:20.549422] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.549442] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e6af100, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221533545, assoc_session_id:3221533545, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306020279117, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3a97f681f8, id:3221533545, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e6af100}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:20.550276] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F02E-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=90, avg_log_submit_to_slide_time=287) [2024-06-25 17:00:20.550281] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1342, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:20.550708] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62749][T1_ReplaySrv6][T1][Y0-0000000000000000-0-0] [lt=7] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:20.550720] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [62749][T1_ReplaySrv6][T1][Y0-0000000000000000-0-0] [lt=9] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1719306008404846, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1719306020550376, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-06-25 17:00:20.550750] INFO [LIB] stat (utility.h:1197) [62749][T1_ReplaySrv6][T1][Y0-0000000000000000-0-0] [lt=10] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=9, this=0x2b3a3f4048f8, extra_info="") [2024-06-25 17:00:20.550757] INFO [LIB] stat (utility.h:1197) [62749][T1_ReplaySrv6][T1][Y0-0000000000000000-0-0] [lt=6] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=57, this=0x2b3a3f404b28, extra_info="") [2024-06-25 17:00:20.555582] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1333) [2024-06-25 17:00:20.560447] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.561065] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABF4538-00061669548FCC2D-0-0] [lt=1][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301546, send_timestamp=1719305998259519) [2024-06-25 17:00:20.561104] INFO [LIB] stat (utility.h:1197) [63319][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=3] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a872048f8, extra_info="") [2024-06-25 17:00:20.561113] INFO [LIB] stat (utility.h:1197) [63319][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=1, this=0x2b3a87204b28, extra_info="") [2024-06-25 17:00:20.567180] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.567187] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.567190] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.567192] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:20.567194] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:20.567249] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:20.582838] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95434][T1004_L0_G1][T1004][YB420ABA45B1-00061646124EF5DA-0-0] [lt=3] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1039, child={server:"10.186.69.177:2882", region:"sys_region", register_time_us:1719306020583260, keepalive_ts:-1}) [2024-06-25 17:00:20.582848] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95434][T1004_L0_G1][T1004][YB420ABA45B1-00061646124EF5DA-0-0] [lt=9] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1039, self="10.191.69.51:2882", child={server:"10.186.69.177:2882", region:"sys_region", register_time_us:1719306020583260, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:20.585958] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.586017] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.588400] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=89000, remain_us=21236579, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=89, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:20.591035] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289111621632, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:20.591349] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1017445, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=221, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:20.593363] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=13141461, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=142, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:20.594152] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.594903] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=12][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:20.594913] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=10][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.594924] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=380966, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=225, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:20.599037] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [63473][T1001_TabletShe][T1001][Y0-0000000000000000-0-0] [lt=5] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-06-25 17:00:20.599047] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:101) [63473][T1001_TabletShe][T1001][Y0-0000000000000000-0-0] [lt=7] [emptytablet] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=4135) [2024-06-25 17:00:20.601082] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=1] will sleep(sleep_us=67000, remain_us=24126093, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=67, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:20.602077] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF_DUMP](palf_id=1007, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1007, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:102232466, max_lsn:85187188644, max_scn:{val:1719306020371565001, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:85187188644}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:85145245604}}, last_submit_lsn:{lsn:85187188522}, last_submit_end_lsn:{lsn:85187188644}, last_submit_log_id:102232466, last_submit_log_pid:1, max_flushed_lsn:{lsn:85187188522}, max_flushed_end_lsn:{lsn:85187188644}, max_flushed_log_pid:1, committed_end_lsn:{lsn:85187188644}, last_slide_log_id:102232466, last_slide_scn:{val:1719306020371565001, v:0}, last_slide_lsn:{lsn:85187188522}, last_slide_end_lsn:{lsn:85187188644}, last_slide_log_pid:1, last_slide_log_accum_checksum:4248084207, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2b3a86d1c530}, [StateMgr]={this:0x2b3a86d202b0, self:"10.191.69.51:2882", palf_id:1007, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, leader:"10.191.69.56:2882", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:102232467, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x2b3a86d26b70, allow_vote:true}, [ConfigMgr]={palf_id:1007, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:1, config_seq:2}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1719306016900033, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1719306016900033, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2b3a86d1dc70}, [ModeMgr]={palf_id:1007, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697227896000, v:0}}, accepted_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697227896000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697227896000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1007, is_inited:true, min_block_max_scn:{val:1718654575889571005, v:0}, min_block_id:1257, min_block_min_scn:{val:1718599209930300002, v:0}, base_lsn_for_block_gc:{lsn:85155950592}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, log_config_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, log_snapshot_meta:{version:1, base_lsn:{lsn:85155950592}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697227896000, v:0}}}, log_meta_storage:{log_tail:{lsn:3248128}, readable_log_tail:{lsn:3248128}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:85223055360}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1007}, block_mgr:{log_dir:"/data/clog/tenant_1002/1007/meta", dir_fd:818, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:63856640, block_header_serialize_buf_:0x2b3a86d22438, flashback_version:0}, log_storage:{log_tail:{lsn:85187188644}, readable_log_tail:{lsn:85187188644}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:85155950592}, min_scn:{val:1719279545564229003, v:0}, curr_block_id:1269, palf_id:1007}, block_mgr:{log_dir:"/data/clog/tenant_1002/1007/log", dir_fd:819, min_block_id:1257, max_block_id:1270, curr_writable_block_id:1269}, logical_block_size_:67104768, curr_block_writable_size_:35866716, block_header_serialize_buf_:0x2b3a86d240b8, flashback_version:0}, palf_epoch:2, last_purge_throttling_ts:-1, this:0x2b3a86d209f0}, [Reconfirm]={palf_id:1007, self:"10.191.69.51:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2b3a86d20430}) [2024-06-25 17:00:20.602188] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=109][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1007, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:20.602194] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF_DUMP](palf_id=1007, self="10.191.69.51:2882", ack_info_list=[]) [2024-06-25 17:00:20.602199] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1007, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:20.602203] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=4] current log_task status(palf_id=1007, self="10.191.69.51:2882", log_id=102232467, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[]) [2024-06-25 17:00:20.605325] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=2099656, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=214, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:20.605898] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [62703][T1_LSMetaCh][T1][YB420ABF4533-0006166172AE6CE3-0-0] [lt=50] fetch root ls success(local_cluster_id=1713323324, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.171:2882", sql_port:2881, role:1, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607828518, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.176:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:4, zone:"zone4", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607829360, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.189.192.96:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:5, zone:"zone5", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607830236, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.51:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607831113, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.56:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["10.186.69.171:2882", "10.191.69.51:2882", "10.191.69.56:2882", "10.186.69.176:2882", "10.189.192.96:2882"]) [2024-06-25 17:00:20.606537] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [63470][T1001_Flush][T1001][Y0-0000000000000000-0-0] [lt=4] ====== traversal_flush timer task ====== [2024-06-25 17:00:20.606547] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:338) [63470][T1001_Flush][T1001][Y0-0000000000000000-0-0] [lt=7] no logstream(ret=0, ls_cnt=0) [2024-06-25 17:00:20.617960] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63299][T1002_LSArchive][T1002][YB420ABF4533-000616423EEE6725-0-0] [lt=7] gc stale ls task succ [2024-06-25 17:00:20.618295] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63484][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=11] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:20.628523] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:20.630572] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.630604] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.630608] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.630610] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.630613] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.630801] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325962B08-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301549, send_timestamp=1719305998329251) [2024-06-25 17:00:20.630848] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-0006164325962B09-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305998329304) [2024-06-25 17:00:20.633309] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.638906] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=5][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.653539] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301568, send_timestamp=1719305998351970) [2024-06-25 17:00:20.654316] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.655566] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABEB8-0-0] [lt=5] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1333, avg_group_log_size=121) [2024-06-25 17:00:20.657193] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.659986] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.660835] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325C3010F-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301547, send_timestamp=1719305998359287) [2024-06-25 17:00:20.660861] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C3010F-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1010, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1342, avg_group_log_size=122) [2024-06-25 17:00:20.660873] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C3010F-0-0] [lt=11] [PALF STAT COMMITTED LOG SIZE](palf_id=1010, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:20.660934] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1010, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1342) [2024-06-25 17:00:20.661118] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABF4533-00061661728FC917-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301546, send_timestamp=1719305998359571) [2024-06-25 17:00:20.665058] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.667237] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=20, this=0x2b3a9537b470, extra_info="") [2024-06-25 17:00:20.667248] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:20.667255] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT APPEND DATA SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1220) [2024-06-25 17:00:20.667261] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a9534ee50, extra_info="id:1006") [2024-06-25 17:00:20.667264] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.667269] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.667271] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.667273] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:20.667276] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:20.667334] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=76, this=0x2b3a953853b0, extra_info="palf_id:1006") [2024-06-25 17:00:20.667349] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:20.667357] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=4, this=0x2b3a953855e0, extra_info="palf_id:1006") [2024-06-25 17:00:20.667479] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABDC060-000616432547F034-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301546, send_timestamp=1719305998365932) [2024-06-25 17:00:20.667517] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F034-0-0] [lt=4] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a9537b6a0, extra_info="palf_id:1006") [2024-06-25 17:00:20.667525] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F034-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=288, this=0x2b3a9537b8d0, extra_info="palf_id:1006") [2024-06-25 17:00:20.667529] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F034-0-0] [lt=4] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=24, avg_log_submit_to_flush_time=92, avg_log_submit_to_slide_time=264) [2024-06-25 17:00:20.674865] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.678128] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62238][pnio1][T1001][YB420ABF4533-00061661718F6542-0-0] [lt=7][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.686074] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.692698] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.694794] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.697609] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.697861] INFO pktc_resp_cb_on_msg (pktc_resp.h:70) [62238][pnio1][T0][Y0-0000000000000000-0-0] [lt=11] PNIO resp cb not found: packet_id=1035876574 [2024-06-25 17:00:20.700548] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=8248720, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=174, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:20.701390] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.702075] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3943266, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=202, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:20.704969] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC094-0-0] [lt=9] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a5d1db470, extra_info="") [2024-06-25 17:00:20.705047] INFO [LIB] stat (utility.h:1197) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=77, this=0x2b3a5d1e53b0, extra_info="palf_id:1") [2024-06-25 17:00:20.705058] INFO [LIB] stat (utility.h:1214) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=6, this=0x2b3a40ffddb0, extra_info="") [2024-06-25 17:00:20.705071] INFO [LIB] stat (utility.h:1197) [62591][T1_LogIOCb0][T1][Y0-0000000000000000-0-0] [lt=10] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x2b3a5d1e55e0, extra_info="palf_id:1") [2024-06-25 17:00:20.706375] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100139][T1004_L0_G0][T1004][YB420ABF4538-00061668E93FAB9A-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.707222] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.710827] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99941][T1004_L0_G0][T1004][YB420ABF4538-00061668E93FAB9A-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.717036] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.720970] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [63472][T1001_ObTimer][T1001][Y0-0000000000000000-0-0] [lt=6] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-06-25 17:00:20.720980] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:230) [63472][T1001_ObTimer][T1001][Y0-0000000000000000-0-0] [lt=7] [tabletchange] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=4135) [2024-06-25 17:00:20.721903] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=68000, remain_us=24005272, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=68, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:20.722736] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABDC060-0006164323C5051C-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5386, fly_ts=22301552, send_timestamp=1719305998421184) [2024-06-25 17:00:20.725512] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100324][T1004_L0_G0][T1004][YB420ABF4538-00061668E80FE1A7-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.726591] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.729306] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100128][T1004_L0_G0][T1004][YB420ABF4538-00061668E80FE1A7-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.730622] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=22, this=0x2bcde0525470, extra_info="") [2024-06-25 17:00:20.730636] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b589fc96e50, extra_info="id:1039") [2024-06-25 17:00:20.730640] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.730672] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.730676] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.730678] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.730681] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.730718] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=71, this=0x2bcde052f3b0, extra_info="palf_id:1039") [2024-06-25 17:00:20.730728] INFO [LIB] stat (utility.h:1214) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbff3b0, extra_info="") [2024-06-25 17:00:20.730735] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=5, this=0x2bcde052f5e0, extra_info="palf_id:1039") [2024-06-25 17:00:20.730847] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-0006164325962B0D-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301547, send_timestamp=1719305998429299) [2024-06-25 17:00:20.730889] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B0D-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2bcde05256a0, extra_info="palf_id:1039") [2024-06-25 17:00:20.730897] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B0D-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=312, this=0x2bcde05258d0, extra_info="palf_id:1039") [2024-06-25 17:00:20.730898] INFO [LIB] stat (utility.h:1197) [99487][T1004_ApplySrv2][T1004][YB420ABF4533-00061660FFBF07DB-0-0] [lt=3] [APPLY STAT CB APPEND COST TIME](cur_stat_count=42, stat_interval=5000000, avg (count/cost)=37, this=0x2b5f26aa0280, extra_info="") [2024-06-25 17:00:20.730906] INFO [LIB] stat (utility.h:1197) [99487][T1004_ApplySrv2][T1004][YB420ABF4533-00061660FFBF07DB-0-0] [lt=7] [APPLY STAT CB IN QUEUE TIME](cur_stat_count=42, stat_interval=5000000, avg (count/cost)=285, this=0x2b5f26aa04b0, extra_info="") [2024-06-25 17:00:20.730909] INFO [LIB] stat (utility.h:1197) [99487][T1004_ApplySrv2][T1004][YB420ABF4533-00061660FFBF07DB-0-0] [lt=2] [APPLY STAT CB WAIT COMMIT TIME](cur_stat_count=42, stat_interval=5000000, avg (count/cost)=0, this=0x2b5f26aa06e0, extra_info="") [2024-06-25 17:00:20.730912] INFO [LIB] stat (utility.h:1197) [99487][T1004_ApplySrv2][T1004][YB420ABF4533-00061660FFBF07DB-0-0] [lt=3] [APPLY STAT CB EXECUTE TIME](cur_stat_count=42, stat_interval=5000000, avg (count/cost)=-536, this=0x2b5f26aa0910, extra_info="") [2024-06-25 17:00:20.730915] INFO [LIB] stat (utility.h:1197) [99487][T1004_ApplySrv2][T1004][YB420ABF4533-00061660FFBF07DB-0-0] [lt=3] [APPLY STAT CB TOTAL TIME](cur_stat_count=42, stat_interval=5000000, avg (count/cost)=-213, this=0x2b5f26aa0b40, extra_info="") [2024-06-25 17:00:20.731306] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=90000, remain_us=21093673, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=90, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:20.733343] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.734878] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.735042] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=4) [2024-06-25 17:00:20.738961] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.739618] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164319BD3E69-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5407, fly_ts=22301547, send_timestamp=1719305998438070) [2024-06-25 17:00:20.745227] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=863567, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=222, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:20.747203] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=12987620, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=143, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:20.747951] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=5][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:20.747960] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:20.748656] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=227234, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=226, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:20.750843] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62690][T1_ReplaySrv0][T1][Y0-0000000000000000-0-0] [lt=10] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:20.750855] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [62690][T1_ReplaySrv0][T1][Y0-0000000000000000-0-0] [lt=8] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1719306008743392, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1719306020750278, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-06-25 17:00:20.758581] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [62160][ServerGTimer][T1004][Y0-0000000000000000-0-0] [lt=7] get_disk_usage(ret=0, capacity(MB):=2073600, used(MB):=1658818) [2024-06-25 17:00:20.759084] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1945897, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=215, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:20.760893] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325C30113-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301545, send_timestamp=1719305998459347) [2024-06-25 17:00:20.761184] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABF4538-00061669548FCC35-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301547, send_timestamp=1719305998459637) [2024-06-25 17:00:20.767293] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.767299] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.767302] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.767304] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:20.767306] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:20.767531] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-000616432547F037-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301558, send_timestamp=1719305998465972) [2024-06-25 17:00:20.768500] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [62265][SerScheQueue1][T1002][YB420ABF4533-0006166046AF0DA6-0-0] [lt=0][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.768512] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62265][SerScheQueue1][T1002][YB420ABF4533-0006166046AF0DA6-0-0] [lt=0] will sleep(sleep_us=11000, remain_us=5652118, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=11, v.err_=-4383, timeout_timestamp=1719306026420630) [2024-06-25 17:00:20.768680] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.771457] INFO [CLOG] do_thread_task_ (ob_remote_fetch_log_worker.cpp:250) [99272][T1004_RFLWorker][T1004][YB420ABF4533-0006164248EE6725-0-0] [lt=7] ObRemoteFetchWorker is running(thread_index=0) [2024-06-25 17:00:20.778171] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62236][pnio1][T1001][YB420ABF4533-00061661718F6543-0-0] [lt=1][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.782481] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:20.786131] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.789831] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.791255] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=12] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289109524480, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:20.792298] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.816961] INFO [COMMON] process (ob_rpc_processor_simple.cpp:2405) [95435][T1004_L0_G19][T1004][YB420ABA45AD-0006164660E3F2CE-0-0] [lt=14] handle ObQueryLSIsValidMember requeset(arg_={tenant_id_:1004, self_addr_:"10.186.69.173:2882", ls_array_:[{id:1039}]}) [2024-06-25 17:00:20.816974] INFO [CLOG] get_member_gc_stat (ob_log_handler.cpp:1509) [95435][T1004_L0_G19][T1004][YB420ABA45AD-0006164660E3F2CE-0-0] [lt=10] get_member_gc_stat(is_valid_member=true, stat=1, member_list=5{server:"10.186.69.173:2882", timestamp:1716928619820263, flag:0}{server:"10.186.69.177:2882", timestamp:1716928774338690, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, learner_list={learner_num:0, learner_array:[]}, addr="10.186.69.173:2882", this={role:1, proposal_id:1, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1039}) [2024-06-25 17:00:20.818910] INFO [COMMON] process (ob_rpc_processor_simple.cpp:2405) [95435][T1004_L0_G19][T1004][YB420ABA45B1-0006164625F87C88-0-0] [lt=13] handle ObQueryLSIsValidMember requeset(arg_={tenant_id_:1004, self_addr_:"10.186.69.177:2882", ls_array_:[{id:1039}]}) [2024-06-25 17:00:20.818916] INFO [CLOG] get_member_gc_stat (ob_log_handler.cpp:1509) [95435][T1004_L0_G19][T1004][YB420ABA45B1-0006164625F87C88-0-0] [lt=5] get_member_gc_stat(is_valid_member=true, stat=1, member_list=5{server:"10.186.69.173:2882", timestamp:1716928619820263, flag:0}{server:"10.186.69.177:2882", timestamp:1716928774338690, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, learner_list={learner_num:0, learner_array:[]}, addr="10.186.69.177:2882", this={role:1, proposal_id:1, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1039}) [2024-06-25 17:00:20.821389] INFO do_work (ob_rl_mgr.cpp:709) [62262][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=9] swc wakeup.(stat_period_=1000000, ready=false) [2024-06-25 17:00:20.826084] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB22-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:20.826332] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC8E1-0-0] [lt=6] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1211) [2024-06-25 17:00:20.827949] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=24] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:20.830685] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.830717] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.830721] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.830724] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.830726] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.830773] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1039, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:20.830927] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B12-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1039, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=27, avg_log_submit_to_flush_time=92, avg_log_submit_to_slide_time=279) [2024-06-25 17:00:20.830945] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325962B13-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301543, send_timestamp=1719305998529402) [2024-06-25 17:00:20.830983] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B13-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1005, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=20, avg_log_submit_to_flush_time=92, avg_log_submit_to_slide_time=277) [2024-06-25 17:00:20.833397] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.833959] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4013, dropped:189, tid:12536}]) [2024-06-25 17:00:20.834598] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.834606] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.834611] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:20.834617] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.834623] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.837810] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [63276][T1002_LockWaitM][T1002][Y0-0000000000000000-0-0] [lt=8] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-06-25 17:00:20.839016] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.843792] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=69000, remain_us=23883384, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=69, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:20.845505] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABF4533-0006166172BEBEAE-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301548, send_timestamp=1719305998543957) [2024-06-25 17:00:20.848299] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [62162][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=5] trace max work area(auto_calc=true, tenant_max_memory_limit=1073741824, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=53687091, max_wa_memory_size=53687091, tmp_max_wa_memory_size=53687091, pre_mem_target=53687091, remain_memory_size=53687091, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=343166976, washable_size=-2, max_workarea_memory_size=53687091, max_tenant_memory_size=730574848, wash_ratio=6, tenant_id=1001) [2024-06-25 17:00:20.848347] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [62162][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=19] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=6710886, calc_global_bound_size=1048576000000, mem_target_=53687091, wa_max_memory_size=53687091, profile_cnt_=0, total_memory_size=0, max_wa_size=53687091, avg_bound_size=6710886, max_bound_size=6710886, min_bound_size_=1048576) [2024-06-25 17:00:20.848357] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [62162][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=9] timer to calc global bound size(ret=0, global_bound_size_=6710886, manual_calc_cnt_=0, drift_size_=0, pre_drift_size=0, wa_max_memory_size=53687091, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1001, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=6710886, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=53687091, auto_calc=true, sql_mem_callback_.get_total_dump_size()=0) [2024-06-25 17:00:20.848382] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [62162][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=8] trace max work area(auto_calc=true, tenant_max_memory_limit=1073741824, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=53687091, max_wa_memory_size=53687091, tmp_max_wa_memory_size=53687091, pre_mem_target=53687091, remain_memory_size=53687091, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=931856384, washable_size=-2, max_workarea_memory_size=53687091, max_tenant_memory_size=141885440, wash_ratio=6, tenant_id=1002) [2024-06-25 17:00:20.848418] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [62162][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=10] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=6710886, calc_global_bound_size=1048576000000, mem_target_=53687091, wa_max_memory_size=53687091, profile_cnt_=0, total_memory_size=0, max_wa_size=53687091, avg_bound_size=6710886, max_bound_size=6710886, min_bound_size_=1048576) [2024-06-25 17:00:20.848424] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [62162][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=5] timer to calc global bound size(ret=0, global_bound_size_=6710886, manual_calc_cnt_=19494, drift_size_=0, pre_drift_size=0, wa_max_memory_size=53687091, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1002, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=6710886, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=53687091, auto_calc=true, sql_mem_callback_.get_total_dump_size()=11170713168) [2024-06-25 17:00:20.848441] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [62162][SqlMemTimer][T1003][Y0-0000000000000000-0-0] [lt=4] trace max work area(auto_calc=true, tenant_max_memory_limit=80530636800, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=4026531840, max_wa_memory_size=4026531840, tmp_max_wa_memory_size=4026531840, pre_mem_target=4026531840, remain_memory_size=4026531840, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=845557760, washable_size=-2, max_workarea_memory_size=4026531840, max_tenant_memory_size=79685079040, wash_ratio=6, tenant_id=1003) [2024-06-25 17:00:20.848470] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [62162][SqlMemTimer][T1003][Y0-0000000000000000-0-0] [lt=7] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=503316480, calc_global_bound_size=1048576000000, mem_target_=4026531840, wa_max_memory_size=4026531840, profile_cnt_=0, total_memory_size=0, max_wa_size=4026531840, avg_bound_size=503316480, max_bound_size=503316480, min_bound_size_=1048576) [2024-06-25 17:00:20.848476] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [62162][SqlMemTimer][T1003][Y0-0000000000000000-0-0] [lt=6] timer to calc global bound size(ret=0, global_bound_size_=503316480, manual_calc_cnt_=0, drift_size_=0, pre_drift_size=0, wa_max_memory_size=4026531840, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1003, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=503316480, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=4026531840, auto_calc=true, sql_mem_callback_.get_total_dump_size()=0) [2024-06-25 17:00:20.848496] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [62162][SqlMemTimer][T1004][Y0-0000000000000000-0-0] [lt=4] trace max work area(auto_calc=true, tenant_max_memory_limit=724775731200, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=36238786560, max_wa_memory_size=36238786560, tmp_max_wa_memory_size=36238786560, pre_mem_target=36238786560, remain_memory_size=36238786560, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=622220120064, washable_size=-2, max_workarea_memory_size=36238786560, max_tenant_memory_size=102555611136, wash_ratio=6, tenant_id=1004) [2024-06-25 17:00:20.848524] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [62162][SqlMemTimer][T1004][Y0-0000000000000000-0-0] [lt=7] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=4529848320, calc_global_bound_size=1048576000000, mem_target_=36238786560, wa_max_memory_size=36238786560, profile_cnt_=0, total_memory_size=0, max_wa_size=36238786560, avg_bound_size=4529848320, max_bound_size=4529848320, min_bound_size_=1048576) [2024-06-25 17:00:20.848530] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [62162][SqlMemTimer][T1004][Y0-0000000000000000-0-0] [lt=6] timer to calc global bound size(ret=0, global_bound_size_=4529848320, manual_calc_cnt_=6994, drift_size_=0, pre_drift_size=6234304, wa_max_memory_size=36238786560, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1004, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=4529848320, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=36238786560, auto_calc=true, sql_mem_callback_.get_total_dump_size()=542226249464) [2024-06-25 17:00:20.849045] INFO [SHARE] refresh_primary_cluster_id (ob_alive_server_tracer.cpp:397) [62163][ServerTracerTim][T0][YB420ABF4533-000616613B2F176C-0-0] [lt=11] refresh primary cluster finish(ret=0, primary_cluster_id_=-1) [2024-06-25 17:00:20.854222] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=8095045, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=175, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:20.855904] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3789436, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=203, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:20.860951] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164325C30117-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301550, send_timestamp=1719305998559401) [2024-06-25 17:00:20.861244] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABF4538-00061669548FCC38-0-0] [lt=9] [PALF STAT LOG TASK TIME](palf_id=1010, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=86, avg_log_submit_to_slide_time=290) [2024-06-25 17:00:20.861248] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1010, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:20.863044] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B71-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1009, self="10.191.69.51:2882", committed size=1098) [2024-06-25 17:00:20.863099] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1098) [2024-06-25 17:00:20.867347] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.867354] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.867356] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.867359] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:20.867361] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:20.867577] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-000616432547F03B-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301550, send_timestamp=1719305998566027) [2024-06-25 17:00:20.875174] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=91000, remain_us=20949804, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=91, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:20.878372] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62244][pnio1][T1001][YB420ABF4533-00061661718F6544-0-0] [lt=10][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:20.879469] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:20.879479] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:20.879486] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999953, timeout_timestamp=1719306050878905) [2024-06-25 17:00:20.879493] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:20.879495] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:20.879500] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:20.879505] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:20.879508] WDIAG [SQL] open (ob_result_set.cpp:157) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=3][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:20.879512] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=3][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:20.879516] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=3][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:20.879521] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.879529] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=7][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:20.879535] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=5][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.879540] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.879542] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:20.879548] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE859-0-0] [lt=3][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:20.879554] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4013, ret=-4013) [2024-06-25 17:00:20.879558] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:20.879563] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:20.879566] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=2][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:20.879571] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=5][errcode=-4013] query failed(ret=-4013, conn=0x2b399eda9760, start=1719306020878905, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:20.879576] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=5][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:20.879581] WDIAG [SHARE.PT] get_by_tenant (ob_persistent_ls_table.cpp:639) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] execute sql failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:20.879603] WDIAG [SHARE.PT] get_by_tenant (ob_ls_table_operator.cpp:252) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] get all ls info by persistent_ls_ failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002) [2024-06-25 17:00:20.879608] WDIAG [SHARE] inner_open_ (ob_ls_table_iterator.cpp:104) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=5][errcode=-4013] fail to get ls infos by tenant(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, inner_table_only=false) [2024-06-25 17:00:20.879612] WDIAG [SHARE] next (ob_ls_table_iterator.cpp:71) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=3][errcode=-4013] fail to open iterator(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.879617] WDIAG [SERVER] build_replica_map_ (ob_tenant_meta_checker.cpp:334) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=5][errcode=-4013] ls table iterator next failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.879622] WDIAG [SERVER] check_ls_table_ (ob_tenant_meta_checker.cpp:214) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=5][errcode=-4013] build replica map from ls table failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", mode=0) [2024-06-25 17:00:20.879627] WDIAG [SERVER] check_ls_table (ob_tenant_meta_checker.cpp:188) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] check ls table failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", mode=0) [2024-06-25 17:00:20.879632] WDIAG [SERVER] runTimerTask (ob_tenant_meta_checker.cpp:44) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE859-0-0] [lt=4][errcode=-4013] fail to check ls meta table(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.883205] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=6][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:20.883215] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:20.883221] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=4][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=9999971, timeout_timestamp=1719306030882838) [2024-06-25 17:00:20.883228] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:20.883234] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:20.883237] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:20.883242] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:20.883244] WDIAG [SQL] open (ob_result_set.cpp:157) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=2][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:20.883247] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:20.883256] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=8][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT /*+ query_timeout(10000000) */ `tenant_id`, `ls_id`, `svr_ip`, `svr_port`, `role` FROM `__all_ls_meta_table` WHERE 1=1 AND tenant_id = 1002 ORDER BY `ls_id` ASC, `svr_ip` ASC, `svr_port` ASC"}) [2024-06-25 17:00:20.883261] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=4][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT /*+ query_timeout(10000000) */ `tenant_id`, `ls_id`, `svr_ip`, `svr_port`, `role` FROM `__all_ls_meta_table` WHERE 1=1 AND tenant_id = 1002 ORDER BY `ls_id` ASC, `svr_ip` ASC, `svr_port` ASC"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:20.883267] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=5][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:20.883273] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=5][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.883276] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.883279] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:20.883286] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7944-0-0] [lt=5][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT /*+ query_timeout(10000000) */ `tenant_id`, `ls_id`, `svr_ip`, `svr_port`, `role` FROM `__all_ls_meta_table` WHERE 1=1 AND tenant_id = 1002 ORDER BY `ls_id` ASC, `svr_ip` ASC, `svr_port` ASC"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:20.883293] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=6][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT /*+ query_timeout(10000000) */ `tenant_id`, `ls_id`, `svr_ip`, `svr_port`, `role` FROM `__all_ls_meta_table` WHERE 1=1 AND tenant_id = 1002 ORDER BY `ls_id` ASC, `svr_ip` ASC, `svr_port` ASC"}, aret=-4013, ret=-4013) [2024-06-25 17:00:20.883296] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT /*+ query_timeout(10000000) */ `tenant_id`, `ls_id`, `svr_ip`, `svr_port`, `role` FROM `__all_ls_meta_table` WHERE 1=1 AND tenant_id = 1002 ORDER BY `ls_id` ASC, `svr_ip` ASC, `svr_port` ASC) [2024-06-25 17:00:20.883301] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=5][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:20.883304] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:20.883307] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] query failed(ret=-4013, conn=0x2b3afefeaeb0, start=1719306020882834, sql=SELECT /*+ query_timeout(10000000) */ `tenant_id`, `ls_id`, `svr_ip`, `svr_port`, `role` FROM `__all_ls_meta_table` WHERE 1=1 AND tenant_id = 1002 ORDER BY `ls_id` ASC, `svr_ip` ASC, `svr_port` ASC) [2024-06-25 17:00:20.883313] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=6][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:20.883316] WDIAG [SERVER] next_tenant_ (ob_iterate_private_virtual_table.cpp:255) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", exec_tenant_id=1001, cur_tenant_id=1002, sql_=SELECT /*+ query_timeout(10000000) */ `tenant_id`, `ls_id`, `svr_ip`, `svr_port`, `role` FROM `__all_ls_meta_table` WHERE 1=1 AND tenant_id = 1002 ORDER BY `ls_id` ASC, `svr_ip` ASC, `svr_port` ASC) [2024-06-25 17:00:20.883322] WDIAG [SERVER] inner_get_next_row (ob_iterate_private_virtual_table.cpp:285) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=6][errcode=-4013] switch to next tenant failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.883327] WDIAG [COMMON] get_next_row (ob_virtual_table_iterator.cpp:373) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=5][errcode=-4013] fail to inner get next row(ret=-4013, scan_param_={tablet_id:{id:1}, ls_id:{id:1}, column_ids:[16, 17, 18, 19, 23], index_id:12239, key_ranges:[{"range":"table_id:12239,group_idx:0,(MIN,MIN,MIN,MIN;MAX,MAX,MAX,MAX)"}], ss_key_ranges:[{"range":"table_id:12239,group_idx:0,(MIN;MAX)"}], range_array_pos:[], timeout:1719306022779587, scan_flag:{scan_order:1, daily_merge:0, rmmb_optimize:0, whole_macro_scan:0, full_row:0, index_back:0, query_stat:0, sql_mode:0, read_latest:0, prewarm:0, join_type:0, use_row_cache:0, use_block_index_cache:0, use_bloomfilter_cache:0, multi_version_minor_merge:0, is_need_feedback:0, use_fuse_row_cache:0, use_fast_agg:0, iter_uncommitted_row:0, ignore_trans_stat:0, is_large_query:0, is_sstable_cut:0, skip_read_lob:0, is_lookup_for_4377:0, is_for_foreign_key_check:0, reserved:0}, sql_mode:4194304, reserved_cell_count:5, schema_version:1713323598189688, query_begin_schema_version:1719272886792320, limit_offset:{offset_:0, limit_:-1}, for_update:false, wait:-1, frozen_version:-1, is_get:false, output_exprs:[{type:"T_REF_COLUMN", datum_meta:{type:"BIGINT", cs_type:"binary", scale:0, precision:20}, obj_meta:{type:"BIGINT", collation:"binary", coercibility:"NUMERIC"}, obj_datum_map:3, flag:2, eval_func:null, eval_batch_func:null, inner_functions:null, inner_func_cnt:0, arg_cnt:0, parent_cnt:1, frame_idx:2, datum_off:112, res_buf_off:500, dyn_buf_header_offset:484, res_buf_len:8, eval_flags_off:0, pvt_skip_off:0, expr_ctx_id:4294967295, extra:0, batch_idx_mask:0, local_session_var_id:-1, this:0x2b3c2d516568}, {type:"T_REF_COLUMN", datum_meta:{type:"BIGINT", cs_type:"binary", scale:0, precision:20}, obj_meta:{type:"BIGINT", collation:"binary", coercibility:"NUMERIC"}, obj_datum_map:3, flag:2, eval_func:null, eval_batch_func:null, inner_functions:null, inner_func_cnt:0, arg_cnt:0, parent_cnt:1, frame_idx:2, datum_off:160, res_buf_off:524, dyn_buf_header_offset:508, res_buf_len:8, eval_flags_off:0, pvt_skip_off:0, expr_ctx_id:4294967295, extra:0, batch_idx_mask:0, local_session_var_id:-1, this:0x2b3c2d516790}, {type:"T_REF_COLUMN", datum_meta:{type:"VARCHAR", cs_type:"utf8mb4_general_ci", scale:-1, precision:-1}, obj_meta:{type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"IMPLICIT"}, obj_datum_map:1, flag:2, eval_func:null, eval_batch_func:null, inner_functions:null, inner_func_cnt:0, arg_cnt:0, parent_cnt:1, frame_idx:2, datum_off:16, res_buf_off:360, dyn_buf_header_offset:344, res_buf_len:46, eval_flags_off:0, pvt_skip_off:0, expr_ctx_id:4294967295, extra:0, batch_idx_mask:0, local_session_var_id:-1, this:0x2b3c2d516118}, {type:"T_REF_COLUMN", datum_meta:{type:"BIGINT", cs_type:"binary", scale:0, precision:20}, obj_meta:{type:"BIGINT", collation:"binary", coercibility:"NUMERIC"}, obj_datum_map:3, flag:2, eval_func:null, eval_batch_func:null, inner_functions:null, inner_func_cnt:0, arg_cnt:0, parent_cnt:1, frame_idx:2, datum_off:64, res_buf_off:476, dyn_buf_header_offset:460, res_buf_len:8, eval_flags_off:0, pvt_skip_off:0, expr_ctx_id:4294967295, extra:0, batch_idx_mask:0, local_session_var_id:-1, this:0x2b3c2d516340}, {type:"T_REF_COLUMN", datum_meta:{type:"BIGINT", cs_type:"binary", scale:0, precision:20}, obj_meta:{type:"BIGINT", collation:"binary", coercibility:"NUMERIC"}, obj_datum_map:3, flag:2, eval_func:null, eval_batch_func:null, inner_functions:null, inner_func_cnt:0, arg_cnt:0, parent_cnt:0, frame_idx:2, datum_off:192, res_buf_off:540, dyn_buf_header_offset:524, res_buf_len:8, eval_flags_off:0, pvt_skip_off:0, expr_ctx_id:4294967295, extra:0, batch_idx_mask:0, local_session_var_id:-1, this:0x2b3c2d516900}], op_filters:NULL, trans_desc:NULL, snapshot:{this:0x2b3a6af7fc10, valid:false, source:0, core:{version:{val:18446744073709551615, v:3}, tx_id:{txid:0}, scn:0}, uncertain_bound:0, snapshot_lsid:{id:-1}, snapshot_ls_role:0, snapshot_acquire_addr:"0.0.0.0:0", parts:[]}, table_param:{table_id:12239, output_projector:[0, 1, 2, 3, 4], aggregate_projector:[], output_sel_mask:[true, true, true, true, true], pad_col_projector:[], main_read_info:{schema_column_count:21, schema_rowkey_cnt:4, rowkey_cnt:6, trans_col_index:-1, seq_read_column_count:4, max_col_index:9, is_oracle_mode:false, cols_index:{rowkey_mode:false, for_memtable:false, array:{ObIArray:[0, 1, 2, 3, 9], data:0x2b3bcca73710, count:5, allocator:0x2b3bccacaeb0, capacity:5, init_cnt:5}}, memtable_cols_index:{rowkey_mode:false, for_memtable:true, array:{ObIArray:[0, 1, 2, 3, 7], data:0x2b3bcca73730, count:5, allocator:0x2b3bccacaeb0, capacity:5, init_cnt:5}}, cols_desc:{ObIArray:[column_id=16 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=17 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=18 {type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"} order=0, column_id=19 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0, column_id=23 {type:"BIGINT", collation:"binary", coercibility:"NUMERIC"} order=0], data:0x2b3bcca736d0, count:5, allocator:0x2b3bccacaeb0, capacity:5, init_cnt:5}, datum_utils:{is_oracle_mode:false, rowkey_cnt:6, col_cnt:7, is_inited:true, is_oracle_mode:false}, cols_param:[{column_id:16, meta_type:{type:"BIGINT", collation:"binary", coercibility:"NUMERIC"}, order:0, accuracy:{length:-1, precision:20, scale:0}, orig_default_value:{"NULL":"NULL"}, cur_default_value:{"NULL":"NULL"}, is_nullable_for_write:false, is_nullable_for_read:false, is_gen_col:false, is_virtual_gen_col:false, is_gen_col_udf_expr:false, is_hidden:false, lob_chunk_size:262144}, {column_id:17, meta_type:{type:"BIGINT", collation:"binary", coercibility:"NUMERIC"}, order:0, accuracy:{length:-1, precision:20, scale:0}, orig_default_value:{"NULL":"NULL"}, cur_default_value:{"NULL":"NULL"}, is_nullable_for_write:false, is_nullable_for_read:false, is_gen_col:false, is_virtual_gen_col:false, is_gen_col_udf_expr:false, is_hidden:false, lob_chunk_size:262144}, {column_id:18, meta_type:{type:"VARCHAR", collation:"utf8mb4_general_ci", coercibility:"INVALID"}, order:0, accuracy:{length:46, precision:-1, scale:-1}, orig_default_value:{"NULL":"NULL"}, cur_default_value:{"NULL":"NULL"}, is_nullable_for_write:false, is_nullable_for_read:false, is_gen_col:false, is_virtual_gen_col:false, is_gen_col_udf_expr:false, is_hidden:false, lob_chunk_size:262144}, {column_id:19, meta_type:{type:"BIGINT", collation:"binary", coercibility:"NUMERIC"}, order:0, accuracy:{length:-1, precision:20, scale:0}, orig_default_value:{"NULL":"NULL"}, cur_default_value:{"NULL":"NULL"}, is_nullable_for_write:false, is_nullable_for_read:false, is_gen_col:false, is_virtual_gen_col:false, is_gen_col_udf_expr:false, is_hidden:false, lob_chunk_size:262144}, {column_id:23, meta_type:{type:"BIGINT", collation:"binary", coercibility:"NUMERIC"}, order:0, accuracy:{length:-1, precision:20, scale:0}, orig_default_value:{"NULL":"NULL"}, cur_default_value:{"NULL":"NULL"}, is_nullable_for_write:false, is_nullable_for_read:false, is_gen_col:false, is_virtual_gen_col:false, is_gen_col_udf_expr:false, is_hidden:false, lob_chunk_size:262144}]}, use_lob_locator:false, rowid_version:0, rowid_projector:[], enable_lob_locator_v2:true}, sample_info:{method:0, percent:"1.000000000000000000e+02", seed:-1, table_id:0, scope:0, force_block:false}, need_scn:false, need_switch_param:false, fb_read_tx_uncommitted:false, external_file_format:}, external_file_location:""}) [2024-06-25 17:00:20.883439] WDIAG [COMMON] get_next_row (ob_virtual_table_iterator.cpp:474) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=112][errcode=-4013] get next row failed(ret=-4013) [2024-06-25 17:00:20.883443] WDIAG [SQL.ENG] get_next_row_with_das (ob_table_scan_op.cpp:1736) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=4][errcode=-4013] get next row from das result failed(ret=-4013) [2024-06-25 17:00:20.883446] WDIAG [SQL.ENG] inner_get_next_row_for_tsc (ob_table_scan_op.cpp:1923) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] fail to get next row from ObNewRowIterator(ret=-4013) [2024-06-25 17:00:20.883449] WDIAG [SQL.ENG] inner_get_next_row_implement (ob_table_scan_op.cpp:1900) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] failed to get next row(ret=-4013) [2024-06-25 17:00:20.883453] WDIAG [SQL.ENG] inner_get_next_row (ob_table_scan_op.cpp:2861) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=4][errcode=-4013] get next row failed(ret=-4013) [2024-06-25 17:00:20.883457] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=4][errcode=-4013] inner get next row failed(ret=-4013, type=3, op="PHY_TABLE_SCAN", op_id=1) [2024-06-25 17:00:20.883462] WDIAG [SQL.ENG] get_next_left_row (ob_hash_join_op.cpp:995) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=5][errcode=-4013] get left row from child failed(ret=-4013) [2024-06-25 17:00:20.883465] WDIAG [SQL.ENG] fill_partition (ob_hash_join_op.cpp:2434) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] get next left row failed(ret=-4013) [2024-06-25 17:00:20.883470] WDIAG [SQL.ENG] split_partition_and_build_hash_table (ob_hash_join_op.cpp:3591) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=4][errcode=-4013] failed split partition(ret=-4013, part_level_=0) [2024-06-25 17:00:20.883475] WDIAG [SQL.ENG] recursive_process (ob_hash_join_op.cpp:3614) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=4][errcode=-4013] failed to build hash table(ret=-4013, part_level_=0) [2024-06-25 17:00:20.883479] WDIAG [SQL.ENG] adaptive_process (ob_hash_join_op.cpp:3646) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] failed to recursive process(ret=-4013, part_level_=0, part_count_=8, hash_table_.nbuckets_=0) [2024-06-25 17:00:20.883483] WDIAG [SQL.ENG] read_right_operate (ob_hash_join_op.cpp:4340) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] build hash table failed(ret=-4013) [2024-06-25 17:00:20.883487] WDIAG [SQL.ENG] next (ob_hash_join_op.cpp:779) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] failed state operation(ret=-4013, state=1) [2024-06-25 17:00:20.883492] WDIAG [SQL.ENG] inner_get_next_row (ob_hash_join_op.cpp:835) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=5][errcode=-4013] fail to get next row(ret=-4013) [2024-06-25 17:00:20.883495] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=2][errcode=-4013] inner get next row failed(ret=-4013, type=7, op="PHY_HASH_JOIN", op_id=0) [2024-06-25 17:00:20.883499] WDIAG [SQL.EXE] get_next_row (ob_execute_result.cpp:147) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] get next row from operator failed(ret=-4013) [2024-06-25 17:00:20.883503] WDIAG [SQL] inner_get_next_row (ob_result_set.cpp:414) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=4][errcode=-4013] get next row from exec result failed(ret=-4013) [2024-06-25 17:00:20.883506] WDIAG [SERVER] open (ob_inner_sql_result.cpp:165) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=2][errcode=-4013] get_next_row failed(ret=-4013, has_tenant_resource()=true) [2024-06-25 17:00:20.883509] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"select /*+query_timeout(2000000)*/ a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;"}) [2024-06-25 17:00:20.883514] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=5][errcode=-4013] execute failed(ret=-4013, tenant_id=1, executor={ObIExecutor:, sql:"select /*+query_timeout(2000000)*/ a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1719272886792320) [2024-06-25 17:00:20.883517] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=3][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:20.883569] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=4][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.883573] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=4][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:20.883576] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:20.883580] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7944-0-0] [lt=2][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"select /*+query_timeout(2000000)*/ a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:20.883585] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62786][BlackListServic][T0][YB420ABF4533-000616616F5E7944-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"select /*+query_timeout(2000000)*/ a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;"}, aret=-4013, ret=-4013) [2024-06-25 17:00:20.883589] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62786][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1, sql=select /*+query_timeout(2000000)*/ a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;) [2024-06-25 17:00:20.883594] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62786][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1) [2024-06-25 17:00:20.883597] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62786][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=2][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1) [2024-06-25 17:00:20.883599] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [62786][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=2][errcode=-4013] query failed(ret=-4013, conn=0x2b3a78d44610, start=1719306020879581, sql=select /*+query_timeout(2000000)*/ a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;) [2024-06-25 17:00:20.883603] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:111) [62786][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:20.883609] WDIAG [STORAGE.TRANS] do_thread_task_ (ob_black_list.cpp:222) [62786][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] fail to execute sql(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", sql=select /*+query_timeout(2000000)*/ a.svr_ip, a.svr_port, a.tenant_id, a.ls_id, a.role, nvl(b.weak_read_scn, 1) as weak_read_scn, nvl(b.migrate_status, 0) as migrate_status, nvl(b.tx_blocked, 0) as tx_blocked from oceanbase.__all_virtual_ls_meta_table a left join oceanbase.__all_virtual_ls_info b on a.svr_ip = b.svr_ip and a.svr_port = b.svr_port and a.tenant_id = b.tenant_id and a.ls_id = b.ls_id;) [2024-06-25 17:00:20.883613] INFO [STORAGE.TRANS] print_stat_ (ob_black_list.cpp:398) [62786][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=3] start to print blacklist info [2024-06-25 17:00:20.883642] INFO [STORAGE.TRANS] run1 (ob_black_list.cpp:194) [62786][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=5] ls blacklist refresh finish(cost_time=4341) [2024-06-25 17:00:20.886193] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.895951] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100251][T1004_L0_G0][T1004][YB420ABF4538-00061668EA3FE9C8-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.897601] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.897610] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.897615] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:20.897621] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.897627] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.898626] WDIAG [SERVER] do_heartbeat_event (ob_heartbeat.cpp:265) [62788][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=11][errcode=0] try reload schema failed(schema_version=1719272886792320, refresh_schema_info={schema_version:1719272886792320, tenant_id:1, sequence_id:1099511641796}, schema_ret=-4023) [2024-06-25 17:00:20.898984] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=709810, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=223, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:20.900387] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100070][T1004_L0_G0][T1004][YB420ABF4538-00061668EA3FE9C8-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:20.900535] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.900544] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.900549] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:20.900555] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.900560] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.901066] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=12833757, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=144, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:20.902247] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=10] re_register_parent reason(palf_id=1007, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306016900033, last_first_register_time_us=1713323714379172, register_time_us=1719306016900033, curr_time_us=1719306020902247) [2024-06-25 17:00:20.902266] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=8] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1007, self="10.191.69.51:2882", reason=1, leader="10.191.69.56:2882", register_time_us=1719306020902256, last_submit_register_req_time_us=1719306020902256, register_parent_reason=1) [2024-06-25 17:00:20.902272] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=5] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1007, self="10.191.69.51:2882") [2024-06-25 17:00:20.902368] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0] will sleep(sleep_us=73522, remain_us=73522, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=227, v.err_=-4383, timeout_timestamp=1719306020975890) [2024-06-25 17:00:20.904455] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.904463] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:20.904467] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:20.904474] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.904478] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.905182] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=4073) [2024-06-25 17:00:20.905926] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A093-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a5d1db6a0, extra_info="palf_id:1") [2024-06-25 17:00:20.905934] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A093-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=835, this=0x2b3a5d1db8d0, extra_info="palf_id:1") [2024-06-25 17:00:20.910318] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.910326] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:20.910330] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:20.910336] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.910340] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.912844] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1792138, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=216, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:20.918653] INFO [SQL.PC] inner_cache_evict (ob_ps_cache.cpp:788) [63118][T1_PsCacheEvict][T1][Y0-0000000000000000-0-0] [lt=13] ps cache evict(tenant_id=1, stmt_id_map_.size()=9, stmt_info_map_.size()=9, op.get_used_size()=19720, get_mem_high()=908989560, expired_stmt_ids.count()=0, closed_stmt_ids.count()=9) [2024-06-25 17:00:20.920105] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.920112] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:20.920117] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:20.920123] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.920127] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.922126] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.922135] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:20.922140] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:20.922147] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.922151] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.922165] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e6ba2c0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221543706, assoc_session_id:3221543706, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306020653376, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2bb3c49201f8, id:3221543706, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e6ba2c0}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:20.922793] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [62718][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=969588866, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=408931, mem_used=767425536) [2024-06-25 17:00:20.926199] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=59339224, this={dio_aligned_buf:{buf_write_offset:594, buf_padding_size:3502, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:872554071, truncate_used_ts:123213227}, log_block_size:67108864, dir_fd:874, io_fd:830}, aligned_buf_len=4096, aligned_buf="", aligned_block_offset=59338752, buf_len=122, total_write_size_=12294, total_write_size_after_dio_=425984, ob_pwrite_used_ts=7501, count=101, each_pwrite_cost=74) [2024-06-25 17:00:20.930735] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:20.930767] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1005, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:20.930776] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:20.930779] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.930782] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.930784] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.930968] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325962B17-0-0] [lt=0][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301549, send_timestamp=1719305998629419) [2024-06-25 17:00:20.930978] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325962B18-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301552, send_timestamp=1719305998629425) [2024-06-25 17:00:20.933020] INFO [STORAGE] runTimerTask (ob_tenant_tablet_scheduler.cpp:192) [62734][T1_InfoPoolResi][T1][Y0-0000000000000000-0-0] [lt=9] InfoPoolResizeTask(cost_ts=5) [2024-06-25 17:00:20.933453] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:20.933475] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF5A0F-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1342, avg_group_log_size=122) [2024-06-25 17:00:20.935108] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=3) [2024-06-25 17:00:20.937833] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.937841] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:20.937845] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:20.937852] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.937856] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.939070] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.950030] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8100FF6A7-0-0] [lt=3] [PALF STAT GROUP LOG INFO](palf_id=1, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=4612, avg_group_log_size=419) [2024-06-25 17:00:20.950062] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1342) [2024-06-25 17:00:20.950172] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [42764][T1002_ComTimerT][T1002][Y0-0000000000000000-0-0] [lt=8] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-06-25 17:00:20.950918] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62749][T1_ReplaySrv6][T1][Y0-0000000000000000-0-0] [lt=3] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:20.955565] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [62818][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=7] current monitor number(seq_=-1) [2024-06-25 17:00:20.956582] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B1A-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=87, avg_log_submit_to_slide_time=849) [2024-06-25 17:00:20.956591] INFO [PALF] read (log_cache.cpp:100) [99517][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1333, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:20.960965] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABDC060-0006164325C3011B-0-0] [lt=5][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301555, send_timestamp=1719305998659410) [2024-06-25 17:00:20.960991] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C3011B-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf16f470, extra_info="") [2024-06-25 17:00:20.961091] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=73, this=0x2b3aaf1793b0, extra_info="palf_id:1010") [2024-06-25 17:00:20.961102] INFO [LIB] stat (utility.h:1214) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=5, this=0x2b3a8ce1d3b0, extra_info="") [2024-06-25 17:00:20.961113] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=6, this=0x2b3aaf1795e0, extra_info="palf_id:1010") [2024-06-25 17:00:20.961285] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABF4533-00061661728FC922-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301551, send_timestamp=1719305998659733) [2024-06-25 17:00:20.961305] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC922-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf16f6a0, extra_info="palf_id:1010") [2024-06-25 17:00:20.961310] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC922-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=294, this=0x2b3aaf16f8d0, extra_info="palf_id:1010") [2024-06-25 17:00:20.962865] INFO [STORAGE.TRANS] alloc (ob_ts_response_handler.cpp:108) [62239][pnio1][T0][YB420ABF4533-0006166171AF65E9-0-0] [lt=8] ts response task statistics(alloc_count=2495631006, free_count=1247864892) [2024-06-25 17:00:20.962949] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=7] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1004, need_refresh=false, gts_local_cache={srr:[mts=1719306020462454], gts:1719306020461982000, latest_srr:[mts=1719306020962943]}) [2024-06-25 17:00:20.963104] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62238][pnio1][T0][YB420ABF4533-0006166171AF65E8-0-0] [lt=7][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:20.963115] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62238][pnio1][T0][YB420ABF4533-0006166171AF65E8-0-0] [lt=10] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:20.963127] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62238][pnio1][T0][YB420ABF4533-0006166171AF65E8-0-0] [lt=6] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306020962773]}) [2024-06-25 17:00:20.964575] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.964584] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:20.964590] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:20.964596] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.964602] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.966580] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=1] will sleep(sleep_us=70000, remain_us=23760596, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=70, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:20.967366] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [62616][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=8] ====== tenant freeze timer task ====== [2024-06-25 17:00:20.967407] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:20.967415] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:20.967418] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:20.967421] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:20.967423] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:20.967624] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-000616432547F03F-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301548, send_timestamp=1719305998666076) [2024-06-25 17:00:20.971558] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:20.971566] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:20.971570] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:20.971576] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:20.971580] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:20.977856] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [63468][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=8][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-06-25 17:00:20.977866] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [63468][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=9] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-06-25 17:00:20.977907] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [63159][T1002_Occam][T1002][YB420ABF4533-000616615CEF8F5C-0-0] [lt=10] for each ls(succ_num=5, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:20.986251] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:20.991464] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=12] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289113718784, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:20.995586] INFO pktc_resp_cb_on_msg (pktc_resp.h:70) [62249][pnio1][T0][Y0-0000000000000000-0-0] [lt=9] PNIO resp cb not found: packet_id=1035876899 [2024-06-25 17:00:21.008018] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=7941249, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=176, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:21.008342] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166943BFC1ED-0-0] [lt=3] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1006, child={server:"10.191.69.56:2882", region:"sys_region", register_time_us:1719306021007284, keepalive_ts:-1}) [2024-06-25 17:00:21.008353] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166943BFC1ED-0-0] [lt=10] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1006, self="10.191.69.51:2882", child={server:"10.191.69.56:2882", region:"sys_region", register_time_us:1719306021007284, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:21.009416] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3635924, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=204, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:21.020076] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=92000, remain_us=20804902, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=92, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:21.022917] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABF4533-000616614DFE8DDA-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301552, send_timestamp=1719305998721364) [2024-06-25 17:00:21.023102] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABF4533-000616614DFE8DDA-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301545, send_timestamp=1719305998721556) [2024-06-25 17:00:21.023138] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABF4533-000616614DFE8DDA-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620693, send_timestamp=1719305970402445) [2024-06-25 17:00:21.027012] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B1B-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=100, avg_log_submit_to_slide_time=831) [2024-06-25 17:00:21.027020] INFO [PALF] read (log_cache.cpp:100) [99516][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:21.027220] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100263][T1004_L0_G0][T1004][YB420ABF4538-00061669267F93E4-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:21.027514] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63474][T1001_LSArchive][T1001][YB420ABF4533-000616423F8E6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:21.029653] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=1][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:228, local_retry_times:228, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:21.029664] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=0][errcode=-4012] failed to close result(close_ret=-4383, ret=-4012) [2024-06-25 17:00:21.029670] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:21.029678] INFO [SERVER] process_final (ob_inner_sql_connection.cpp:734) [63185][T1002_TeRec][T1002][YB420ABF4533-00061660F99EA319-0-0] [lt=6] slow inner sql(last_ret=-4012, sql={ObIExecutor:, sql:"START TRANSACTION"}, process_time=30053780) [2024-06-25 17:00:21.029683] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63185][T1002_TeRec][T1002][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"START TRANSACTION"}, aret=-4012, ret=-4012) [2024-06-25 17:00:21.029690] WDIAG [SERVER] start_transaction_inner (ob_inner_sql_connection.cpp:1101) [63185][T1002_TeRec][T1002][YB420ABF4533-00061660F99EA319-0-0] [lt=5][errcode=-4012] start transaction failed(ret=-4012, tenant_id=1001, with_snap_shot=false) [2024-06-25 17:00:21.029700] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63185][T1002_TeRec][T1002][YB420ABF4533-00061660F99EA319-0-0] [lt=3][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1001) [2024-06-25 17:00:21.029704] WDIAG [SERVER] start_transaction (ob_inner_sql_connection.cpp:1060) [63185][T1002_TeRec][T1002][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4012] start_transaction failed(ret=-4012, tenant_id=1001, with_snap_shot=false) [2024-06-25 17:00:21.029708] WDIAG [COMMON.MYSQLP] start_transaction (ob_mysql_transaction.cpp:61) [63185][T1002_TeRec][T1002][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4012] fail to start transaction(ret=-4012, tenant_id=1001, with_snapshot=false) [2024-06-25 17:00:21.029732] WDIAG [COMMON.MYSQLP] start (ob_mysql_transaction.cpp:82) [63185][T1002_TeRec][T1002][YB420ABF4533-00061660F99EA319-0-0] [lt=4][errcode=-4012] failed to start transaction(ret=-4012, tenant_id=1001, with_snapshot=false) [2024-06-25 17:00:21.030789] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:21.030819] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:21.030824] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.030826] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.030829] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.031023] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC060-0006164325962B1C-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301546, send_timestamp=1719305998729477) [2024-06-25 17:00:21.031101] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99991][T1004_L0_G0][T1004][YB420ABF4538-00061669267F93E4-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:21.033246] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [63297][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=5] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-06-25 17:00:21.033256] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [63297][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=7] [tabletchange] task check ls(ls->get_ls_id()={id:1006}, tablet_persist_trigger=0) [2024-06-25 17:00:21.033262] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [63297][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=6] [tabletchange] task check ls(ls->get_ls_id()={id:1007}, tablet_persist_trigger=0) [2024-06-25 17:00:21.033266] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [63297][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [tabletchange] task check ls(ls->get_ls_id()={id:1008}, tablet_persist_trigger=0) [2024-06-25 17:00:21.033269] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [63297][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=3] [tabletchange] task check ls(ls->get_ls_id()={id:1009}, tablet_persist_trigger=0) [2024-06-25 17:00:21.033272] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [63297][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=2] [tabletchange] task check ls(ls->get_ls_id()={id:1010}, tablet_persist_trigger=0) [2024-06-25 17:00:21.033277] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:228) [63297][T1002_ObTimer][T1002][Y0-0000000000000000-0-0] [lt=4] [tabletchange] succeed to gc_tablet(ret=0, ret="OB_SUCCESS", ls_cnt=5, times=3741) [2024-06-25 17:00:21.033508] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:21.035240] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301564, send_timestamp=1719305998733676) [2024-06-25 17:00:21.037409] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.037418] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.037424] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.037431] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.037436] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.039123] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=5][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.040551] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.040561] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:21.040567] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:21.040573] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.040578] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.043847] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1044) [99249][T1004_FreInfoRe][T1004][Y0-0000000000000000-0-0] [lt=16] success to update min reserved snapshot(reserved_snapshot=1719304212419817000, duration=1800, snapshot_gc_ts_=1719306012419817000) [2024-06-25 17:00:21.043905] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99249][T1004_FreInfoRe][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT GROUP LOG INFO](palf_id=1039, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=11, avg_log_batch_cnt=1, total_group_log_size=1332, avg_group_log_size=121) [2024-06-25 17:00:21.043914] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99249][T1004_FreInfoRe][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT APPEND DATA SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1342) [2024-06-25 17:00:21.044144] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABDC060-0006164325962B1F-0-0] [lt=15][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301556, send_timestamp=1719305998742588) [2024-06-25 17:00:21.044371] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99249][T1004_FreInfoRe][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT APPEND DATA SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1342) [2024-06-25 17:00:21.044447] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1332) [2024-06-25 17:00:21.044581] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-0006164325962B20-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301551, send_timestamp=1719305998743029) [2024-06-25 17:00:21.044686] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1071) [99249][T1004_FreInfoRe][T1004][Y0-0000000000000000-0-0] [lt=8] update reserved snapshot finished(cost_ts=841, reserved_snapshot=1719304212419817000) [2024-06-25 17:00:21.046379] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.046389] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:21.046394] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:21.046401] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.046406] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.052847] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=555948, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=224, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:21.052968] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [99392][T1004_ReqMemEvi][T1004][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1004, evict_high_mem_level=20669530113, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=8345600, mem_used=15148855296) [2024-06-25 17:00:21.053603] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63292][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7290, mem_used=12478464) [2024-06-25 17:00:21.054890] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=12679933, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=145, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:21.055871] INFO [LIB] stat (utility.h:1214) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=21, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbfec70, extra_info="") [2024-06-25 17:00:21.056519] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.056527] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.056532] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:21.056540] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.056544] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.062487] INFO [RPC] process (ob_batch_processor.cpp:106) [99937][T1004_L0_G0][T1004][YB420ABF4538-00061669533F590A-0-0] [lt=9] batch rpc statistics(clog_batch_nodelay_cnt=0, clog_batch_cnt=0, trx_batch_cnt=1, sql_batch_cnt=0) [2024-06-25 17:00:21.062744] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=38266) [2024-06-25 17:00:21.062791] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [99501][T1004_ApplySrv3][T1004][YB420ABF4533-0006165A169F181D-0-0] [lt=8] rpc statistics(total_trans_msg_count=3, total_batch_msg_count=3) [2024-06-25 17:00:21.063120] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [99516][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=161956757504, cur_mem_hold=55475961856, attr_.tenant_id_=1004) [2024-06-25 17:00:21.063132] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [99516][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=9] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1719305980686539, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1719306021062630, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-06-25 17:00:21.063151] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B79-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1009, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=9, avg_log_batch_cnt=0, total_group_log_size=1098, avg_group_log_size=122) [2024-06-25 17:00:21.063977] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F042-0-0] [lt=9] [PALF STAT LOG TASK TIME](palf_id=1009, self="10.191.69.51:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=76, avg_log_submit_to_slide_time=839) [2024-06-25 17:00:21.063985] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT HOT CACHE HIT RATE](palf_id=1009, read_size=1098, hit_cnt=9, read_cnt=9, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:21.064199] INFO [LIB] stat (utility.h:1197) [99504][T1004_ApplySrv3][T1004][YB420ABF4533-00061653B4EF8FED-0-0] [lt=4] long trans statistics(cur_stat_count=12, stat_interval=60000000, avg (count/cost)=0, this=0x563ae0f8f590, extra_info="") [2024-06-25 17:00:21.066376] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1638606, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=217, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:21.067437] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.067444] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.067447] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.067450] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:21.067453] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:21.070039] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.070048] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.070053] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.070059] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.070065] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.072901] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.072909] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:21.072914] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.072919] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.072925] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.076714] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.076722] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:21.076726] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.076731] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.076735] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.078292] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62247][pnio1][T1001][YB420ABF4533-00061661718F6545-0-0] [lt=7][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:21.082480] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.082487] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:21.082491] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.082496] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.082502] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.086308] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.090411] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=71000, remain_us=23636764, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=71, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:21.092211] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.092219] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:21.092222] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.092228] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.092237] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.094866] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616430D6F51D6-0-0] [lt=7] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1006, child={server:"10.189.192.96:2882", region:"sys_region", register_time_us:1719305998793297, keepalive_ts:-1}) [2024-06-25 17:00:21.094875] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616430D6F51D6-0-0] [lt=8] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1006, self="10.191.69.51:2882", child={server:"10.189.192.96:2882", region:"sys_region", register_time_us:1719305998793297, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:21.106416] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.106426] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:21.106431] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.106437] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.106442] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.108630] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [62479][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=9] new mysql sessid created(conn.sessid_=3221540637, support_ssl=false) [2024-06-25 17:00:21.108653] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [62479][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=10] sm conn init succ(conn.sessid_=3221540637, sess.client_addr_="10.191.69.51:48032") [2024-06-25 17:00:21.108661] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [62479][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=6] accept one succ(*s={this:0x2b9ea1ffc030, session_id:3221540637, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:907}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:21.108832] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [62479][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x2b9ea1ffc030, session_id:3221540637, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:907}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:21.108843] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [62479][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=9] kill and revert session(conn.sessid_=3221540637, proxy_sessid=0, server_id=1, ret=0) [2024-06-25 17:00:21.108848] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:918) [62479][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=4] sock ref clean, do destroy(*s={this:0x2b9ea1ffc030, session_id:3221540637, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:907}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:21.108855] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:223) [62479][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=6] mark session id unused(conn.sessid_=3221540637) [2024-06-25 17:00:21.108860] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [62479][sql_nio0][T0][Y0-0000000000000000-0-0] [lt=4] connection close(sessid=3221540637, proxy_sessid=0, tenant_id=0, server_id=1, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=false, ret=0, trace_id=Y0-0000000000000000-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:0], disconnect_state=0) [2024-06-25 17:00:21.109986] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.109995] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.109999] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.110005] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.110010] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.124362] WDIAG [STORAGE.TRANS] process_ (ob_gti_rpc.h:178) [62244][pnio1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=12][errcode=0] gti rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:21.126249] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB2E-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:21.127297] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F043-0-0] [lt=5] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=89, avg_log_submit_to_slide_time=832) [2024-06-25 17:00:21.128118] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [63342][T1001_Occam][T1001][YB420ABF4533-000616615CFF8F5C-0-0] [lt=7] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:21.130841] INFO eloop_run (eloop.c:143) [62232][pnio1][T0][Y0-0000000000000000-0-0] [lt=7] PNIO [ratelimit] time: 1719306021130840, bytes: 29137867390586, bw: 0.706778 MB/s, add_ts: 1000541, add_bytes: 741511 [2024-06-25 17:00:21.130844] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:21.130874] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:21.130878] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.130880] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.130883] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.131073] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164325962B2A-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305998829529) [2024-06-25 17:00:21.131073] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164325962B29-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301550, send_timestamp=1719305998829522) [2024-06-25 17:00:21.133563] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:21.134229] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4383, dropped:823, tid:63185}]) [2024-06-25 17:00:21.135171] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=7] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=2) [2024-06-25 17:00:21.139177] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=5][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.142465] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62430][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142525] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62427][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142583] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62426][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142637] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62431][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142642] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62420][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142642] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62417][RpcIO][T0][Y0-0000000000000000-0-0] [lt=5] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142648] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62432][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142671] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62424][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142740] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62422][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142750] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62421][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142805] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62423][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142813] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62415][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.142917] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62419][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.143028] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62418][RpcIO][T0][Y0-0000000000000000-0-0] [lt=5] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.143131] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62416][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.143209] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62425][RpcIO][T0][Y0-0000000000000000-0-0] [lt=5] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.143246] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62429][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.143548] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62428][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.143862] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.143871] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.143875] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.143881] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.143885] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.147582] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100575][T1003_L0_G0][T1003][YB420ABF4538-000616694DDF523D-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1006}}) [2024-06-25 17:00:21.147600] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100575][T1003_L0_G0][T1004][YB420ABF4538-000616694DDF523D-0-0] [lt=10] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306021062794002, v:0}, arg={tenant_id:1004, ls_id:{id:1006}}, result={tenant_id:1004, ls_id:{id:1006}, cur_readable_scn:{val:1719306021062794002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:21.148171] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SYS](tenant_id=1, io_status=sys_group_name: MICRO_BLOCK_CACHE_IO, mode: read, size: 4096.00, iops: 11.18, rt: 76.83) [2024-06-25 17:00:21.148181] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS CONFIG](tenant_id=1, ref_cnt=2, io_config={group_num_:0, memory_limit_:8589934592, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:4}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=12763552, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:21.148216] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=16] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 0.93, rt: 101.00) [2024-06-25 17:00:21.148222] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SYS](tenant_id=500, io_status=sys_group_name: INSPECT_BAD_BLOCK_IO, mode: read, size: 2097152.00, iops: 0.93, rt: 776.00) [2024-06-25 17:00:21.148226] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:21.148285] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=11] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148292] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148295] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148298] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148302] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148306] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148309] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148312] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148315] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148320] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148323] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148326] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148329] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148334] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148337] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.148340] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:21.150001] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF793C-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86d1d470, extra_info="") [2024-06-25 17:00:21.150011] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF793C-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:21.150017] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF793C-0-0] [lt=5] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:21.150098] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=69, this=0x2b3a86d273b0, extra_info="palf_id:1007") [2024-06-25 17:00:21.150124] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3a86d275e0, extra_info="palf_id:1007") [2024-06-25 17:00:21.150125] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8100FF6AC-0-0] [lt=11] [PALF STAT COMMITTED LOG SIZE](palf_id=1, self="10.191.69.51:2882", committed size=4073) [2024-06-25 17:00:21.150310] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F044-0-0] [lt=4] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a86d1d6a0, extra_info="palf_id:1007") [2024-06-25 17:00:21.150315] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F044-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=293, this=0x2b3a86d1d8d0, extra_info="palf_id:1007") [2024-06-25 17:00:21.152912] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=8][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:224, local_retry_times:224, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.152923] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.152928] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.152930] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:21.152937] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:21.154955] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=6][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:145, local_retry_times:145, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.154965] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.154970] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.154972] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:21.154977] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:21.156679] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B2B-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1333) [2024-06-25 17:00:21.161073] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-0006164325C30123-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301551, send_timestamp=1719305998859521) [2024-06-25 17:00:21.161366] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABF4533-00061661728FC92A-0-0] [lt=4][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301546, send_timestamp=1719305998859820) [2024-06-25 17:00:21.161473] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=8][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:71, local_retry_times:71, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.161482] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.161487] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.161489] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:21.161495] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:21.161755] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:21.161764] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:21.161768] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e7528c0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021107900, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.161788] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=19][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e7528c0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021107900, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.161799] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:21.161806] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:21.161810] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:21.161814] WDIAG [SQL] open (ob_result_set.cpp:157) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:21.161817] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:21.161821] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:21.161825] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=177, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.161832] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=6] will sleep(sleep_us=100000, remain_us=7787436, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=177, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:21.163026] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:21.163037] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:21.163042] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6ab35960, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021108954, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.163063] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=20][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6ab35960, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021108954, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.163075] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999998, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:21.163081] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:21.163086] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:21.163088] WDIAG [SQL] open (ob_result_set.cpp:157) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:21.163092] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:21.163096] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}) [2024-06-25 17:00:21.163100] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, retry_cnt=205, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.163107] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=6] will sleep(sleep_us=100000, remain_us=3482234, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=205, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:21.166114] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:21.166125] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:21.166128] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a678fc8c0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021112116, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.166148] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=19][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a678fc8c0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021112116, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.166159] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:21.166166] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:21.166171] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:21.166174] WDIAG [SQL] open (ob_result_set.cpp:157) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:21.166177] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:21.166182] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:21.166187] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=93, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.166192] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4] will sleep(sleep_us=93000, remain_us=20658786, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=93, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:21.166439] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=9][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:217, local_retry_times:217, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.166450] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.166453] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.166457] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:21.166464] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:21.167226] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.167241] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=14][errcode=-4013] [2024-06-25 17:00:21.167246] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:21.167252] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.167257] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.167262] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:21.167267] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:21.167272] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:21.167275] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:21.167278] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:21.167281] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:21.167286] WDIAG [SQL.ENG] inner_get_next_batch_for_tsc (ob_table_scan_op.cpp:2015) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:21.167289] WDIAG [SQL.ENG] inner_get_next_batch (ob_table_scan_op.cpp:2000) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=2][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:21.167292] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] get next batch failed(ret=-4383, eval_ctx={batch_idx:0, batch_size:16, max_batch_size:16, frames_:0x2b5cd7214380}, id=2, op_name="PHY_TABLE_SCAN") [2024-06-25 17:00:21.167302] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1535) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4383] get next batch failed(ret=-4383) [2024-06-25 17:00:21.167305] WDIAG [SQL.ENG] get_next_row_vectorizely (ob_operator.cpp:1463) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] Failed to get_next_row vectorizedly(ret=-4383) [2024-06-25 17:00:21.167309] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_transmit_op.cpp:27) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] get next row from child failed(ret=-4383) [2024-06-25 17:00:21.167313] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] inner get next row failed(ret=-4383, type=14, op="PHY_DIRECT_TRANSMIT", op_id=1) [2024-06-25 17:00:21.167318] WDIAG [SQL.EXE] sync_send_result (ob_remote_executor_processor.cpp:283) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:21.167322] WDIAG [SQL.EXE] execute_remote_plan (ob_remote_executor_processor.cpp:469) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] sync send result failed(ret=-4383) [2024-06-25 17:00:21.167328] WDIAG [SQL.EXE] process (ob_remote_executor_processor.cpp:969) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] execute remote plan failed(ret=-4383, local_tenant_schema_version=1718209168713032, exec_ctx_.get_task_exec_ctx().get_query_tenant_begin_schema_version()=1718209168713032) [2024-06-25 17:00:21.167527] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.167533] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.167537] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.167540] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:21.167542] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:21.167606] INFO [LIB] stat (utility.h:1214) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=18, stat_interval=1000000, avg (count/cost)=6, this=0x2b3a8ce1cc70, extra_info="") [2024-06-25 17:00:21.167753] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-000616432547F046-0-0] [lt=14][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301556, send_timestamp=1719305998866196) [2024-06-25 17:00:21.168190] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [63199][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=8] ====== tenant freeze timer task ====== [2024-06-25 17:00:21.172180] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.172190] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:21.172195] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:21.172202] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.172206] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.172413] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:21.172421] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:21.173176] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3B-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:21.173184] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3B-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.173190] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3B-0-0] [lt=3][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.174157] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63309][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[{id:1010}, {id:1008}, {id:1006}, {id:1007}, {id:1009}]) [2024-06-25 17:00:21.178330] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62248][pnio1][T1001][YB420ABF4533-00061661718F6546-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.186363] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.191701] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289089040384, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:21.206749] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:21.206758] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:21.206762] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78ddfa80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021152090, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.206779] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=16][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78ddfa80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021152090, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.206791] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:21.206798] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:21.206801] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:21.206804] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:21.206807] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:21.206811] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:21.206815] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=225, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.206820] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=401974, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=225, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:21.208774] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:21.208783] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:21.208786] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a69353820, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021154161, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.208803] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a69353820, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021154161, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.208814] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:21.208820] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:21.208824] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:21.208827] WDIAG [SQL] open (ob_result_set.cpp:157) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:21.208829] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:21.208832] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:21.208837] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=146, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.208843] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=12525980, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=146, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:21.209787] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.209795] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:21.209800] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:21.209805] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.209811] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.215224] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:21.215237] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=12][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:21.215240] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6d6e8060, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021160484, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.215257] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6d6e8060, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021160484, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.215269] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:21.215275] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:21.215278] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:21.215282] WDIAG [SQL] open (ob_result_set.cpp:157) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:21.215284] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=1][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:21.215287] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:21.215291] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=72, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.215297] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5] will sleep(sleep_us=72000, remain_us=23511879, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=72, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:21.219914] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=6] ====== check clog disk timer task ====== [2024-06-25 17:00:21.219924] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=7] get_disk_usage(ret=0, capacity(MB):=32384, used(MB):=25883) [2024-06-25 17:00:21.219935] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=5] cannot_recycle_log_size statistics(cannot_recycle_log_size=229976632, threshold=10187125555, need_update_checkpoint_scn=false) [2024-06-25 17:00:21.220116] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:21.220127] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:21.220131] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e691700, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021165739, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.220152] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=20][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e691700, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021165739, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.220163] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:21.220170] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:21.220174] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:21.220178] WDIAG [SQL] open (ob_result_set.cpp:157) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:21.220183] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=5][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:21.220186] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:21.220192] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=6][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=218, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.220198] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=1484783, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=218, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:21.225313] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABDC060-0006164323C50521-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5386, fly_ts=22301555, send_timestamp=1719305998923757) [2024-06-25 17:00:21.226536] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC8F1-0-0] [lt=3] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86de7470, extra_info="") [2024-06-25 17:00:21.226549] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC8F1-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1333, avg_group_log_size=121) [2024-06-25 17:00:21.226618] INFO [LIB] stat (utility.h:1197) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=69, this=0x2b3a86df13b0, extra_info="palf_id:1008") [2024-06-25 17:00:21.226630] INFO [LIB] stat (utility.h:1214) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3a8ce1daf0, extra_info="") [2024-06-25 17:00:21.226645] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a86df15e0, extra_info="palf_id:1008") [2024-06-25 17:00:21.227416] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F047-0-0] [lt=6] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a86de76a0, extra_info="palf_id:1008") [2024-06-25 17:00:21.227423] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F047-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=838, this=0x2b3a86de78d0, extra_info="palf_id:1008") [2024-06-25 17:00:21.229046] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=22] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:21.230596] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [62719][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=31] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1, server_version={version:{val:1719306020948144000, v:0}, total_part_count:1, valid_inner_part_count:1, valid_user_part_count:0}, server_version_delta=282450, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="10.186.69.171:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=58514745, succ_cluster_heartbeat_count=58514745, cluster_heartbeat_interval=50000, local_cluster_version={val:1719305497271300000, v:0}, local_cluster_delta=523959294, force_self_check=false, weak_read_refresh_interval=100000) [2024-06-25 17:00:21.230909] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:21.230942] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:21.230947] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.230950] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.230952] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.230986] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1332) [2024-06-25 17:00:21.231139] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-0006164325962B2D-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301551, send_timestamp=1719305998929588) [2024-06-25 17:00:21.231166] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABDC060-0006164325962B2E-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301551, send_timestamp=1719305998929615) [2024-06-25 17:00:21.232283] WDIAG [SHARE] inner_get_ls_status_ (ob_ls_status_operator.cpp:955) [100438][T1003_DupTbLeas][T1003][Y0-0000000000000000-0-0] [lt=8][errcode=-4018] ls not exist in __all_ls_status table(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-06-25 17:00:21.232320] WDIAG [SHARE] get_duplicate_ls_status_info (ob_ls_status_operator.cpp:674) [100438][T1003_DupTbLeas][T1003][Y0-0000000000000000-0-0] [lt=10][errcode=-4018] fail to inner get ls status info(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sql=SELECT * FROM __all_ls_status where tenant_id = 1003 and flag like "%DUPLICATE%", tenant_id=1003, exec_tenant_id=1, need_member_list=false) [2024-06-25 17:00:21.232426] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=5] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647478, offset=60297206}, end_cursor:ObLogCursor{file_id=100, log_id=27647479, offset=60297447}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbac050, len:4106, log_data_len:241, seq:27647478, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:21.232589] WDIAG [SHARE] inner_get_ls_status_ (ob_ls_status_operator.cpp:955) [63442][T1001_DupTbLeas][T1001][Y0-0000000000000000-0-0] [lt=7][errcode=-4018] ls not exist in __all_ls_status table(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-06-25 17:00:21.232618] WDIAG [SHARE] get_duplicate_ls_status_info (ob_ls_status_operator.cpp:674) [63442][T1001_DupTbLeas][T1001][Y0-0000000000000000-0-0] [lt=9][errcode=-4018] fail to inner get ls status info(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sql=SELECT * FROM __all_ls_status where tenant_id = 1001 and flag like "%DUPLICATE%", tenant_id=1001, exec_tenant_id=1, need_member_list=false) [2024-06-25 17:00:21.233488] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF5A17-0-0] [lt=9] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=13, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b61c69470, extra_info="") [2024-06-25 17:00:21.233505] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8014-0-0] [lt=10] push gts task success(*task={:""}) [2024-06-25 17:00:21.233577] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=13, stat_interval=1000000, avg (count/cost)=69, this=0x2b3b61c733b0, extra_info="palf_id:1006") [2024-06-25 17:00:21.233599] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=13, stat_interval=1000000, avg (count/cost)=7, this=0x2b3b61c735e0, extra_info="palf_id:1006") [2024-06-25 17:00:21.233618] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:21.233825] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B2F-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=13, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b61c696a0, extra_info="palf_id:1006") [2024-06-25 17:00:21.233830] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B2F-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=13, stat_interval=1000000, avg (count/cost)=286, this=0x2b3b61c698d0, extra_info="palf_id:1006") [2024-06-25 17:00:21.236095] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8014-0-0] [lt=9] rpc statistics(total_trans_msg_count=2, total_batch_msg_count=0) [2024-06-25 17:00:21.236677] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=14] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647479, offset=60297447}, end_cursor:ObLogCursor{file_id=100, log_id=27647480, offset=60297696}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fba8050, len:3865, log_data_len:249, seq:27647479, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:21.239233] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=5][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.241030] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=8] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647480, offset=60297696}, end_cursor:ObLogCursor{file_id=100, log_id=27647481, offset=60297945}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fba4050, len:3616, log_data_len:249, seq:27647480, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:21.241135] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647481, offset=60297945}, end_cursor:ObLogCursor{file_id=100, log_id=27647482, offset=60298194}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbc0050, len:3367, log_data_len:249, seq:27647481, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:21.241233] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647482, offset=60298194}, end_cursor:ObLogCursor{file_id=100, log_id=27647483, offset=60298443}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbb4050, len:3118, log_data_len:249, seq:27647482, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:21.245015] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [99399][T1004_LSArchive][T1004][YB420ABF4533-00061642492E6725-0-0] [lt=8] gc stale ls task succ [2024-06-25 17:00:21.247244] INFO eloop_run (eloop.c:143) [62250][pnio2][T0][Y0-0000000000000000-0-0] [lt=9] PNIO [ratelimit] time: 1719306021247244, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007619, add_bytes: 0 [2024-06-25 17:00:21.248021] INFO [SQL.PC] inner_cache_evict (ob_ps_cache.cpp:788) [104904][T1004_PsCacheEv][T1004][Y0-0000000000000000-0-0] [lt=11] ps cache evict(tenant_id=1004, stmt_id_map_.size()=998, stmt_info_map_.size()=998, op.get_used_size()=3183455, get_mem_high()=16307453880, expired_stmt_ids.count()=0, closed_stmt_ids.count()=998) [2024-06-25 17:00:21.249839] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1961) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(dag_cnt=0, map_size=0) [2024-06-25 17:00:21.249849] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1971) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(running_dag_net_map_size=0, blocking_dag_net_list_size=0) [2024-06-25 17:00:21.249853] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_COMPACTION_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.249859] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(priority="PRIO_HA_HIGH", low_limit=8, up_limit=8, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.249862] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_COMPACTION_MID", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.249865] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_HA_MID", low_limit=5, up_limit=5, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.249870] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_COMPACTION_LOW", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.249872] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(priority="PRIO_HA_LOW", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.249877] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(priority="PRIO_DDL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.249879] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(priority="PRIO_DDL_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.249882] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_TTL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.249885] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.249891] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.249894] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.249897] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.249901] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.249905] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.249908] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.249911] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.249915] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.249918] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.249921] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.249926] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.249929] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:21.249932] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:21.249936] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:21.249939] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:21.249941] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:21.249944] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:21.249947] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:21.249951] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:21.249955] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.249958] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.249960] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.249965] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.249968] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.249970] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.249974] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.249978] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.249980] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.249983] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.249987] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.249990] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.249993] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.249997] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250000] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.250003] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250005] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.250009] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250012] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.250015] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250018] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.250022] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250026] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.250030] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250035] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.250037] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250040] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.250043] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250045] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.250049] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250052] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-06-25 17:00:21.250055] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-06-25 17:00:21.250058] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-06-25 17:00:21.250062] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-06-25 17:00:21.250065] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.250068] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.250072] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.250075] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.250077] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.250081] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.250084] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.250087] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.250090] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.250095] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.250097] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.250100] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.250103] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.250107] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.250110] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.250113] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.250117] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.250121] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.250123] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.250126] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250129] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.250131] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250135] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.250139] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250142] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.250147] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250149] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.250152] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250155] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.250159] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250162] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.250164] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250167] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.250170] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250175] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.250178] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250180] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.250183] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.250187] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, dag_count=0) [2024-06-25 17:00:21.250190] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, scheduled_task_count=0) [2024-06-25 17:00:21.250192] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, dag_count=0) [2024-06-25 17:00:21.250195] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, scheduled_task_count=0) [2024-06-25 17:00:21.250200] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, dag_count=0) [2024-06-25 17:00:21.250203] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-06-25 17:00:21.250208] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, dag_count=0) [2024-06-25 17:00:21.250212] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-06-25 17:00:21.250214] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, dag_count=0) [2024-06-25 17:00:21.250217] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, scheduled_task_count=0) [2024-06-25 17:00:21.250219] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_MIGRATION", dag_net_count=0) [2024-06-25 17:00:21.250224] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=4] dump_dag_status[DAG_NET](type="DAG_NET_PREPARE_MIGRATION", dag_net_count=0) [2024-06-25 17:00:21.250225] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=1] dump_dag_status[DAG_NET](type="DAG_NET_COMPLETE_MIGRATION", dag_net_count=0) [2024-06-25 17:00:21.250228] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER", dag_net_count=0) [2024-06-25 17:00:21.250236] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=8] dump_dag_status[DAG_NET](type="DAG_NET_BACKUP", dag_net_count=0) [2024-06-25 17:00:21.250238] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_RESTORE", dag_net_count=0) [2024-06-25 17:00:21.250240] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_TYPE_BACKUP_CLEAN", dag_net_count=0) [2024-06-25 17:00:21.250242] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER_BACKFILL_TX", dag_net_count=0) [2024-06-25 17:00:21.250245] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1991) [100422][T1003_DagSchedu][T1003][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(total_worker_cnt=43, total_running_task_cnt=0, work_thread_num=43, scheduled_task_cnt=0) [2024-06-25 17:00:21.259255] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=6][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:93, local_retry_times:93, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.259269] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=12][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.259273] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.259277] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:21.259283] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:21.261128] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164325C30127-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301551, send_timestamp=1719305998959576) [2024-06-25 17:00:21.261893] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:177, local_retry_times:177, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.261903] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.261906] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.261909] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:21.261916] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:21.263166] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:205, local_retry_times:205, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.263179] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=11][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.263183] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.263187] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:21.263193] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:21.267581] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.267588] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.267591] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.267593] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:21.267596] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:21.267792] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-000616432547F04A-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305998966248) [2024-06-25 17:00:21.268818] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.268832] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.271893] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.275689] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.278331] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [63377][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=10] ====== tenant freeze timer task ====== [2024-06-25 17:00:21.278346] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [63377][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=9][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-06-25 17:00:21.278404] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62237][pnio1][T1001][YB420ABF4533-00061661718F6547-0-0] [lt=7][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.281477] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.285806] INFO [STORAGE] schedule_all_tablets_minor (ob_tenant_tablet_scheduler.cpp:511) [99405][T1004_MergeLoop][T1004][YB420ABF4533-000616424EBF2990-0-0] [lt=9] start schedule all tablet minor merge(minor_ls_tablet_iter_={ls_idx:-1, ls_ids:[{id:1039}, {id:1005}, {id:1006}, {id:1007}, {id:1008}], tablet_idx:0, tablet_ids_.count():0, schedule_tablet_cnt:0, max_batch_tablet_cnt:50000, report_scn_flag:false}) [2024-06-25 17:00:21.286429] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.287331] INFO [STORAGE.BLKMGR] inspect_bad_block (ob_block_manager.cpp:1712) [62325][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=13] check macro block(block_info={ref_cnt:1, is_free:false, access_time:1718732035210907}, time_interval=573986076422) [2024-06-25 17:00:21.287364] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:72, local_retry_times:72, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.287374] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.287378] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.287380] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:21.287387] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:21.291227] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.296557] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [99249][T1004_FreInfoRe][T1004][Y0-0000000000000000-0-0] [lt=6] get all snapshots(ret=0, snapshots.count()=3, snapshots=[{snapshot_type:1, snapshot_scn:{val:1717463693595282000, v:0}, schema_version:1717463690222192, tenant_id:1004, tablet_id:208717, comment:null}, {snapshot_type:1, snapshot_scn:{val:1719194468064666000, v:0}, schema_version:1719194439686752, tenant_id:1004, tablet_id:209217, comment:null}, {snapshot_type:1, snapshot_scn:{val:1719194468064666000, v:0}, schema_version:1719194439686752, tenant_id:1004, tablet_id:1152921504606854157, comment:null}]) [2024-06-25 17:00:21.298369] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [62699][T1_LockWaitMgr][T1][Y0-0000000000000000-0-0] [lt=10] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-06-25 17:00:21.302008] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.302020] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.302028] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:21.302034] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:21.302039] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:21.302042] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:21.302046] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:21.302050] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:21.302053] WDIAG [SQL.ENG] inner_get_next_batch_for_tsc (ob_table_scan_op.cpp:2015) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:21.302056] WDIAG [SQL.ENG] inner_get_next_batch (ob_table_scan_op.cpp:2000) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:21.302061] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4383] get next batch failed(ret=-4383, eval_ctx={batch_idx:0, batch_size:16, max_batch_size:16, frames_:0x2b5bd0e13580}, id=2, op_name="PHY_TABLE_SCAN") [2024-06-25 17:00:21.302069] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1535) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4383] get next batch failed(ret=-4383) [2024-06-25 17:00:21.302072] WDIAG [SQL.ENG] get_next_row_vectorizely (ob_operator.cpp:1463) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] Failed to get_next_row vectorizedly(ret=-4383) [2024-06-25 17:00:21.302075] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_transmit_op.cpp:27) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] get next row from child failed(ret=-4383) [2024-06-25 17:00:21.302079] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4383] inner get next row failed(ret=-4383, type=14, op="PHY_DIRECT_TRANSMIT", op_id=1) [2024-06-25 17:00:21.302085] WDIAG [SQL.EXE] sync_send_result (ob_remote_executor_processor.cpp:283) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:21.302088] WDIAG [SQL.EXE] execute_remote_plan (ob_remote_executor_processor.cpp:469) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4383] sync send result failed(ret=-4383) [2024-06-25 17:00:21.302093] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e68be20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221645809, assoc_session_id:3221645809, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306021035322, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2bc4745b1bf8, id:3221645809, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e68be20}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:21.302129] WDIAG [SQL.EXE] process (ob_remote_executor_processor.cpp:969) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=32][errcode=-4383] execute remote plan failed(ret=-4383, local_tenant_schema_version=1718209168713032, exec_ctx_.get_task_exec_ctx().get_query_tenant_begin_schema_version()=1718209168713032) [2024-06-25 17:00:21.306088] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A096-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1, self="10.191.69.51:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=90, avg_log_submit_to_slide_time=829) [2024-06-25 17:00:21.306096] INFO [PALF] read (log_cache.cpp:100) [62745][T1_ReplaySrv2][T1][Y0-0000000000000000-0-0] [lt=8] [PALF STAT HOT CACHE HIT RATE](palf_id=1, read_size=4194, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:21.306646] INFO [STORAGE] runTimerTask (ob_tenant_tablet_scheduler.cpp:135) [99405][T1004_MergeLoop][T1004][YB420ABF4533-000616424EBF2990-0-0] [lt=13] MergeLoopTask(cost_ts=20854) [2024-06-25 17:00:21.306890] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:225, local_retry_times:225, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.306901] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.306904] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.306908] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:21.306912] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:21.308898] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.308907] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=7][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:146, local_retry_times:146, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.308917] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.308920] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.308923] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:21.308928] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:21.312931] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:21.312941] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:21.312945] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a678f5240, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021258312, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.312964] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=19][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a678f5240, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021258312, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.312975] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:21.312982] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:21.312986] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:21.312990] WDIAG [SQL] open (ob_result_set.cpp:157) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:21.312993] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:21.312998] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:21.313002] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=94, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.313009] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5] will sleep(sleep_us=94000, remain_us=20511970, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=94, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:21.315549] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:21.315558] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:21.315569] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=7633698, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=178, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:21.316901] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3328439, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=206, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:21.326344] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB36-0-0] [lt=3] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b74f470, extra_info="") [2024-06-25 17:00:21.326432] INFO [LIB] stat (utility.h:1197) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=15] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=80, this=0x2b3b5b7593b0, extra_info="palf_id:1007") [2024-06-25 17:00:21.326441] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:21.326450] INFO [LIB] stat (utility.h:1214) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3afcbffaf0, extra_info="") [2024-06-25 17:00:21.326463] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=9, this=0x2b3b5b7595e0, extra_info="palf_id:1007") [2024-06-25 17:00:21.327190] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B31-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b5b74f6a0, extra_info="palf_id:1007") [2024-06-25 17:00:21.327195] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B31-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=838, this=0x2b3b5b74f8d0, extra_info="palf_id:1007") [2024-06-25 17:00:21.330961] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:21.330994] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:21.330999] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.331002] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.331004] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.331081] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1005, self="10.191.69.51:2882", committed size=3174) [2024-06-25 17:00:21.331203] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325962B32-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301547, send_timestamp=1719305999029656) [2024-06-25 17:00:21.333670] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] tx gc loop thread is running(MTL_ID()=1) [2024-06-25 17:00:21.333677] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] try gc retain ctx [2024-06-25 17:00:21.333741] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B34-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=12, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=84, avg_log_submit_to_slide_time=285) [2024-06-25 17:00:21.333749] INFO [PALF] read (log_cache.cpp:100) [99515][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=37] [PALF STAT HOT CACHE HIT RATE](palf_id=1006, read_size=38546, hit_cnt=12, read_cnt=12, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:21.334148] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=2] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1, *ls_ptr={ls_meta:{tenant_id:1, ls_id:{id:1}, ls_create_status:1, clog_checkpoint_scn:{val:1719272888853692000, v:0}, clog_base_lsn:{lsn:35095793664}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:1719285609440317000, v:0}, all_id_meta:{id_meta:[{limited_id:1719306035353935000, latest_log_ts:{val:1719306018786724001, v:0}}, {limited_id:80000001, latest_log_ts:{val:1719298896011036001, v:0}}, {limited_id:2660000001, latest_log_ts:{val:1719295281004326001, v:0}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a437f0030, is_in_stop_state_:false, is_inited_:true, id_:1}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-06-25 17:00:21.334180] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=30] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:21.335239] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=1) [2024-06-25 17:00:21.339287] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.339686] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.339695] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.339700] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.339706] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.339711] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.339714] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.341092] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.341108] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=73000, remain_us=23386068, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=73, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:21.342575] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.343707] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63467][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=11] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7270, mem_used=12478464) [2024-06-25 17:00:21.348072] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABF4533-0006166172BEBEB8-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301552, send_timestamp=1719305999046519) [2024-06-25 17:00:21.360734] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:21.360741] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=7][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:21.360745] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78dc5bc0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021306684, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.360758] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=13][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78dc5bc0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021306684, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.360769] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=10][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:21.360773] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:21.360776] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:21.360780] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:21.360782] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:21.360785] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:21.360789] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=226, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.360794] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=248000, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=226, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:21.361186] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164325C3012B-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301556, send_timestamp=1719305999059629) [2024-06-25 17:00:21.362683] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62460][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.362689] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.362703] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=12372120, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=147, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:21.362854] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62454][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.362943] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62456][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.363056] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62459][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.363277] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B85-0-0] [lt=4] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf09d470, extra_info="") [2024-06-25 17:00:21.363325] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62457][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.363359] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=60, this=0x2b3aaf0a73b0, extra_info="palf_id:1009") [2024-06-25 17:00:21.363378] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=5, this=0x2b3aaf0a75e0, extra_info="palf_id:1009") [2024-06-25 17:00:21.363436] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62458][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.363903] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62455][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.364143] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F04D-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf09d6a0, extra_info="palf_id:1009") [2024-06-25 17:00:21.364148] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F04D-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=840, this=0x2b3aaf09d8d0, extra_info="palf_id:1009") [2024-06-25 17:00:21.364826] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [62725][T1_LSArchiveMgr][T1][YB420ABF4533-000616423A1E6725-0-0] [lt=5] gc stale ls task succ [2024-06-25 17:00:21.366519] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62470][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367246] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62471][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367317] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62443][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367446] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62466][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367451] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62448][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367587] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62442][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367627] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62440][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367640] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.367647] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.367649] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.367652] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:21.367654] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:21.367657] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62438][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367657] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62441][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367659] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62436][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367678] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62464][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367695] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62439][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367725] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62437][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367778] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62462][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367781] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62467][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367795] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62434][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367857] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62463][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367859] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-000616432547F04E-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301544, send_timestamp=1719305999066314) [2024-06-25 17:00:21.367930] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62469][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=9] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367932] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62473][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=4] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.367982] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62447][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.368063] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62461][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.368068] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62450][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.368240] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62465][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.368263] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62468][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.368273] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62446][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.368325] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62451][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.368368] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62444][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.368372] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62435][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.368411] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62445][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.368644] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62452][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.368650] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62449][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.372990] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62475][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:21.373807] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=1331174, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=219, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:21.378412] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62240][pnio1][T1001][YB420ABF4533-00061661718F6548-0-0] [lt=10][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.378447] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100254][T1004_L0_G0][T1004][YB420ABF4534-00061645055D619B-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:21.380596] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [62735][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[{id:1}]) [2024-06-25 17:00:21.382217] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100271][T1004_L0_G0][T1004][YB420ABF4534-00061645055D619B-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:21.385109] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [63298][T1002_TabletShe][T1002][Y0-0000000000000000-0-0] [lt=6] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-06-25 17:00:21.385120] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:99) [63298][T1002_TabletShe][T1002][Y0-0000000000000000-0-0] [lt=6] [emptytablet] succeed to change tablet to empty shell(ret=0, ret="OB_SUCCESS", ls_cnt=5, times=4134) [2024-06-25 17:00:21.386492] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.388313] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABF4533-0006166172BEBEBD-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301549, send_timestamp=1719305999086763) [2024-06-25 17:00:21.391920] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=11] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289111621632, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:21.395884] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [62914][T1_L0_G0][T1][YB420ABA45AB-000616C80E7FB3F0-0-0] [lt=3] [RS_MGR] force set rs list(master_rs="10.186.69.171:2882") [2024-06-25 17:00:21.395894] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [62914][T1_L0_G0][T1][YB420ABA45AB-000616C80E7FB3F0-0-0] [lt=7] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"10.191.69.51:2882", server_id:1, rs_addr:"10.186.69.171:2882", rs_server_status:1, epoch_id:1}, hb_response={zone:"zone1", server:"10.191.69.51:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1713323604978888, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=10) [2024-06-25 17:00:21.407070] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=6][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.408409] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.414515] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.416071] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=5] ====== check clog disk timer task ====== [2024-06-25 17:00:21.416083] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=7] get_disk_usage(ret=0, capacity(MB):=5529, used(MB):=4371) [2024-06-25 17:00:21.416098] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=5] cannot_recycle_log_size statistics(cannot_recycle_log_size=288391690, threshold=1739461755, need_update_checkpoint_scn=false) [2024-06-25 17:00:21.416964] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=9][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.417378] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.421142] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.426892] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.427299] INFO [LIB] stat (utility.h:1197) [99518][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=12] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=32, stat_interval=1000000, avg (count/cost)=7, this=0x2b3ae6e6c8f8, extra_info="") [2024-06-25 17:00:21.427309] INFO [LIB] stat (utility.h:1197) [99518][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=10] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=32, stat_interval=1000000, avg (count/cost)=20, this=0x2b3ae6e6cb28, extra_info="") [2024-06-25 17:00:21.427403] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164325210E42-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=330, fly_ts=22301549, send_timestamp=1719305999125853) [2024-06-25 17:00:21.430993] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:21.431027] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:21.431032] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.431034] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.431037] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.431224] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABDC060-0006164325962B37-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301555, send_timestamp=1719305999129668) [2024-06-25 17:00:21.431259] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-0006164325962B38-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301553, send_timestamp=1719305999129706) [2024-06-25 17:00:21.432831] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1961) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(dag_cnt=0, map_size=0) [2024-06-25 17:00:21.432842] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1971) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=10] dump_dag_status(running_dag_net_map_size=0, blocking_dag_net_list_size=0) [2024-06-25 17:00:21.432846] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_COMPACTION_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.432853] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(priority="PRIO_HA_HIGH", low_limit=8, up_limit=8, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.432855] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(priority="PRIO_COMPACTION_MID", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.432858] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_HA_MID", low_limit=5, up_limit=5, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.432861] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_COMPACTION_LOW", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.432865] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_HA_LOW", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.432868] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_DDL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.432871] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(priority="PRIO_DDL_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.432873] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(priority="PRIO_TTL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:21.432877] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.432881] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.432885] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.432889] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.432891] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.432894] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.432898] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.432901] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.432905] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.432908] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.432912] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:21.432915] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:21.432918] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:21.432922] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:21.432925] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:21.432928] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:21.432932] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:21.432935] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:21.432937] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:21.432941] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:21.432945] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.432948] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.432951] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.432954] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.432957] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.432961] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.432964] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.432967] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.432969] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.432973] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.432976] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.432979] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.432983] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.432986] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.432989] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.432992] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.432996] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.433001] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433003] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.433006] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433009] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.433014] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433016] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.433019] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433022] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.433025] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433028] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.433031] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433034] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:21.433038] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433041] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-06-25 17:00:21.433043] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-06-25 17:00:21.433046] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-06-25 17:00:21.433049] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-06-25 17:00:21.433053] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.433056] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.433059] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.433062] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.433065] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.433070] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.433072] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.433075] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.433078] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.433081] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.433086] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.433089] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.433091] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.433094] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.433097] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.433101] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.433104] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:21.433106] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:21.433109] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.433113] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433116] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.433119] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433123] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.433126] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433128] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.433131] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433135] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.433138] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433141] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.433143] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433147] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.433150] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433154] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.433156] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433159] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.433162] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433166] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:21.433168] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:21.433171] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, dag_count=0) [2024-06-25 17:00:21.433174] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, scheduled_task_count=0) [2024-06-25 17:00:21.433178] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, dag_count=0) [2024-06-25 17:00:21.433181] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, scheduled_task_count=0) [2024-06-25 17:00:21.433183] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, dag_count=0) [2024-06-25 17:00:21.433186] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-06-25 17:00:21.433189] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, dag_count=0) [2024-06-25 17:00:21.433193] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-06-25 17:00:21.433197] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, dag_count=0) [2024-06-25 17:00:21.433200] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, scheduled_task_count=0) [2024-06-25 17:00:21.433203] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_MIGRATION", dag_net_count=0) [2024-06-25 17:00:21.433206] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_PREPARE_MIGRATION", dag_net_count=0) [2024-06-25 17:00:21.433209] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_COMPLETE_MIGRATION", dag_net_count=0) [2024-06-25 17:00:21.433212] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER", dag_net_count=0) [2024-06-25 17:00:21.433215] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_BACKUP", dag_net_count=0) [2024-06-25 17:00:21.433217] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_RESTORE", dag_net_count=0) [2024-06-25 17:00:21.433220] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TYPE_BACKUP_CLEAN", dag_net_count=0) [2024-06-25 17:00:21.433223] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER_BACKFILL_TX", dag_net_count=0) [2024-06-25 17:00:21.433226] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1991) [62670][T1_DagScheduler][T1][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(total_worker_cnt=43, total_running_task_cnt=0, work_thread_num=43, scheduled_task_cnt=0) [2024-06-25 17:00:21.433728] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:21.433769] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B39-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=38546) [2024-06-25 17:00:21.436646] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.439341] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.443191] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC061-00061643965B6191-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=330, fly_ts=50620695, send_timestamp=1719305970822495) [2024-06-25 17:00:21.447618] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABA45B0-00061644F03C36DC-0-0] [lt=7] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1008}}) [2024-06-25 17:00:21.447635] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABA45B0-00061644F03C36DC-0-0] [lt=10] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306021140696002, v:0}, arg={tenant_id:1004, ls_id:{id:1008}}, result={tenant_id:1004, ls_id:{id:1008}, cur_readable_scn:{val:1719306021140696002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:21.451212] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62690][T1_ReplaySrv0][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:21.454432] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.455561] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABA45AB-000616C80A1FC7F7-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1007}}) [2024-06-25 17:00:21.455573] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABA45AB-000616C80A1FC7F7-0-0] [lt=7] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306021046986003, v:0}, arg={tenant_id:1004, ls_id:{id:1007}}, result={tenant_id:1004, ls_id:{id:1007}, cur_readable_scn:{val:1719306021046986003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:21.456010] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABED8-0-0] [lt=9] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b6d5470, extra_info="") [2024-06-25 17:00:21.456097] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=68, this=0x2b3b5b6df3b0, extra_info="palf_id:1008") [2024-06-25 17:00:21.456120] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3b5b6df5e0, extra_info="palf_id:1008") [2024-06-25 17:00:21.456858] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B3A-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b5b6d56a0, extra_info="palf_id:1008") [2024-06-25 17:00:21.456866] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B3A-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=847, this=0x2b3b5b6d58d0, extra_info="palf_id:1008") [2024-06-25 17:00:21.458164] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.458171] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:21.458176] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.458181] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.458187] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.460731] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=1] will sleep(sleep_us=95000, remain_us=20364247, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=95, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:21.460860] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:226, local_retry_times:226, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.460869] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.460873] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:21.460875] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:21.460879] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:21.461337] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABDC060-0006164325C3012F-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301624, send_timestamp=1719305999159712) [2024-06-25 17:00:21.463248] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=12] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-06-25 17:00:21.463271] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=9] gts statistics(tenant_id=1002, gts_rpc_cnt=25, get_gts_cache_cnt=400, get_gts_with_stc_cnt=16, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-06-25 17:00:21.463284] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=8] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1003, need_refresh=false, gts_local_cache={srr:[mts=1719306021238532], gts:1719306021238499000, latest_srr:[mts=1719306021463279]}) [2024-06-25 17:00:21.463604] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62247][pnio1][T0][YB420ABF4533-0006166171AF65ED-0-0] [lt=12][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:21.463615] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62247][pnio1][T0][YB420ABF4533-0006166171AF65ED-0-0] [lt=10] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:21.463629] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62247][pnio1][T0][YB420ABF4533-0006166171AF65ED-0-0] [lt=7] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306021463262]}) [2024-06-25 17:00:21.464071] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.464081] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.467695] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.467703] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.467705] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.467708] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:21.467710] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:21.467930] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-000616432547F052-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301553, send_timestamp=1719305999166376) [2024-06-25 17:00:21.468015] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=74000, remain_us=23259161, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=74, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:21.469338] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=7479929, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=179, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:21.470565] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3174775, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=207, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:21.473863] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.473873] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=9][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.478513] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62246][pnio1][T1001][YB420ABF4533-00061661718F6549-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.486507] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.486696] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [100471][T1003_LSArchive][T1003][YB420ABF4533-0006164249CE6725-0-0] [lt=8] gc stale ls task succ [2024-06-25 17:00:21.488251] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.491730] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.493602] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [100481][T1003_HAService][T1003][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:21.499845] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [99409][T1004_HAService][T1004][Y0-0000000000000000-0-0] [lt=7] start do ls ha handler(ls_id_array_=[{id:1006}, {id:1005}, {id:1007}, {id:1039}, {id:1008}]) [2024-06-25 17:00:21.514641] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:21.514651] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:21.514655] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78dee780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021460021, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.514672] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=16][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78dee780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306021460021, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:21.514683] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=10][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:21.514688] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:21.514692] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:21.514695] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:21.514699] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:21.514702] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:21.514707] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=227, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.514713] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5] will sleep(sleep_us=94082, remain_us=94082, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=227, v.err_=-4383, timeout_timestamp=1719306021608794) [2024-06-25 17:00:21.516502] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=12218321, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=148, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:21.525361] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=5] ====== check clog disk timer task ====== [2024-06-25 17:00:21.525371] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=9] get_disk_usage(ret=0, capacity(MB):=230400, used(MB):=0) [2024-06-25 17:00:21.525379] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=5] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=72477573120, need_update_checkpoint_scn=false) [2024-06-25 17:00:21.526756] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.526801] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1211) [2024-06-25 17:00:21.527621] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1177360, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=220, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:21.531010] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:21.531040] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=14, stat_interval=1000000, avg (count/cost)=18, this=0x2b3b5a817470, extra_info="") [2024-06-25 17:00:21.531049] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT APPEND COST TIME](cur_stat_count=14, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5a884e50, extra_info="id:1005") [2024-06-25 17:00:21.531059] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:21.531062] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.531065] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.531067] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.531123] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=14, stat_interval=1000000, avg (count/cost)=73, this=0x2b3b5a8213b0, extra_info="palf_id:1005") [2024-06-25 17:00:21.531140] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT FLUSH CB COST TIME](cur_stat_count=14, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8215e0, extra_info="palf_id:1005") [2024-06-25 17:00:21.531255] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-0006164325962B3C-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301557, send_timestamp=1719305999229697) [2024-06-25 17:00:21.531330] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B3D-0-0] [lt=3] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=14, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8176a0, extra_info="palf_id:1005") [2024-06-25 17:00:21.531339] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B3D-0-0] [lt=9] [PALF STAT LOG LIFE TIME](cur_stat_count=14, stat_interval=1000000, avg (count/cost)=291, this=0x2b3b5a8178d0, extra_info="palf_id:1005") [2024-06-25 17:00:21.533780] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:21.535304] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=7] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=0) [2024-06-25 17:00:21.536059] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [100464][T1003_ReqMemEvi][T1003][Y0-0000000000000000-0-0] [lt=9] sql audit evict task end(request_manager_->get_tenant_id()=1003, evict_high_mem_level=1932735283, evict_high_size_level=90000, evict_batch_count=0, elapse_time=1, size_used=83009, mem_used=195495936) [2024-06-25 17:00:21.538228] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.538953] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC061-00061643952B83BE-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=50620691, send_timestamp=1719305970918262) [2024-06-25 17:00:21.539396] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.550050] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:444) [62236][pnio1][T1002][YB420ABF4533-0006166170CFAE7C-0-0] [lt=11][errcode=-4016] tenant weak read service cluster heartbeat cost too much time(tenant_id_=1002, delta=525138, last_succ_cluster_heartbeat_tstamp_=1719306021024912) [2024-06-25 17:00:21.550258] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48FB-0-0] [lt=4][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=1768, pos=316, bitmap=NULL) [2024-06-25 17:00:21.550292] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F054-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=88, avg_log_submit_to_slide_time=281) [2024-06-25 17:00:21.550298] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:21.550431] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48FB-0-0] [lt=9][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=434, pos=303, bitmap=NULL) [2024-06-25 17:00:21.550623] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48FB-0-0] [lt=5][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=14, len=174, pos=153, bitmap=NULL) [2024-06-25 17:00:21.550778] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48FB-0-0] [lt=4][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=3059, pos=315, bitmap=NULL) [2024-06-25 17:00:21.553932] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.556154] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:21.561618] INFO [LIB] stat (utility.h:1197) [63266][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=3] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a872048f8, extra_info="") [2024-06-25 17:00:21.561626] INFO [LIB] stat (utility.h:1197) [63266][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=7] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=1, this=0x2b3a87204b28, extra_info="") [2024-06-25 17:00:21.567748] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.567756] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.567758] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.567761] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:21.567764] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:21.567812] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1098) [2024-06-25 17:00:21.567981] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABDC060-000616432547F056-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301551, send_timestamp=1719305999266429) [2024-06-25 17:00:21.569372] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=8][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.586567] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.592141] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=12] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289111621632, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:21.592932] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.595918] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=75000, remain_us=23131257, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=75, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:21.607384] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Cache wash clean map node details(ret=0, clean_node_count=9, clean_time=0, clean_start_pos=173140790, clean_num=402653) [2024-06-25 17:00:21.608281] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [62703][T1_LSMetaCh][T1][YB420ABF4533-0006166172AE6CE4-0-0] [lt=48] fetch root ls success(local_cluster_id=1713323324, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.171:2882", sql_port:2881, role:1, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607828518, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.176:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:4, zone:"zone4", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607829360, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.189.192.96:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:5, zone:"zone5", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607830236, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.51:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607831113, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.56:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["10.186.69.171:2882", "10.191.69.51:2882", "10.191.69.56:2882", "10.186.69.176:2882", "10.189.192.96:2882"]) [2024-06-25 17:00:21.608857] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:227, local_retry_times:227, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:21.608868] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.609509] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=96000, remain_us=20215470, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=96, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:21.618016] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63299][T1002_LSArchive][T1002][YB420ABF4533-000616423EEE6725-0-0] [lt=7] gc stale ls task succ [2024-06-25 17:00:21.618363] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63484][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=7] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:21.619668] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [63284][T1002_LCLSender][T1002][Y0-0000000000000000-0-0] [lt=12] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=30358025, total_destructed_detector=30358025, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x2b3a8a7e42b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-06-25 17:00:21.623056] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=7326211, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=180, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:21.624299] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=3021042, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=208, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:21.630124] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=22] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:21.631061] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:21.631092] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:21.631096] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.631098] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.631101] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.631342] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164325962B42-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301551, send_timestamp=1719305999329791) [2024-06-25 17:00:21.633826] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:21.639449] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.640543] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95434][T1004_L0_G1][T1004][YB420ABF4538-000616694DAF5333-0-0] [lt=3] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1039, child={server:"10.191.69.56:2882", region:"sys_region", register_time_us:1719306021639802, keepalive_ts:-1}) [2024-06-25 17:00:21.640552] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95434][T1004_L0_G1][T1004][YB420ABF4538-000616694DAF5333-0-0] [lt=9] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1039, self="10.191.69.51:2882", child={server:"10.191.69.56:2882", region:"sys_region", register_time_us:1719306021639802, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:21.647409] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.650238] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.651317] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62690][T1_ReplaySrv0][T1][Y0-0000000000000000-0-0] [lt=11] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:21.651356] INFO [LIB] stat (utility.h:1197) [62690][T1_ReplaySrv0][T1][Y0-0000000000000000-0-0] [lt=8] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a3f4048f8, extra_info="") [2024-06-25 17:00:21.651363] INFO [LIB] stat (utility.h:1197) [62690][T1_ReplaySrv0][T1][Y0-0000000000000000-0-0] [lt=6] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=45, this=0x2b3a3f404b28, extra_info="") [2024-06-25 17:00:21.653996] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.656133] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABEE0-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:21.659821] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.661368] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-0006164325C30137-0-0] [lt=1][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301563, send_timestamp=1719305999359804) [2024-06-25 17:00:21.661398] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C30137-0-0] [lt=11] [PALF STAT GROUP LOG INFO](palf_id=1010, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:21.661409] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C30137-0-0] [lt=11] [PALF STAT COMMITTED LOG SIZE](palf_id=1010, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:21.661480] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1010, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:21.661671] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABF4533-00061661728FC93E-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301554, send_timestamp=1719305999360116) [2024-06-25 17:00:21.662679] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:228, local_retry_times:228, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:21.662689] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=0][errcode=-4012] failed to close result(close_ret=-4383, ret=-4012) [2024-06-25 17:00:21.662695] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:21.662701] INFO [SERVER] process_final (ob_inner_sql_connection.cpp:734) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=4] slow inner sql(last_ret=-4012, sql={ObIExecutor:, sql:"START TRANSACTION"}, process_time=30053900) [2024-06-25 17:00:21.662706] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"START TRANSACTION"}, aret=-4012, ret=-4012) [2024-06-25 17:00:21.662711] WDIAG [SERVER] start_transaction_inner (ob_inner_sql_connection.cpp:1101) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4012] start transaction failed(ret=-4012, tenant_id=1001, with_snap_shot=false) [2024-06-25 17:00:21.662721] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1001) [2024-06-25 17:00:21.662725] WDIAG [SERVER] start_transaction (ob_inner_sql_connection.cpp:1060) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4012] start_transaction failed(ret=-4012, tenant_id=1001, with_snap_shot=false) [2024-06-25 17:00:21.662728] WDIAG [COMMON.MYSQLP] start_transaction (ob_mysql_transaction.cpp:61) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4012] fail to start transaction(ret=-4012, tenant_id=1001, with_snapshot=false) [2024-06-25 17:00:21.662751] WDIAG [COMMON.MYSQLP] start (ob_mysql_transaction.cpp:82) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=4][errcode=-4012] failed to start transaction(ret=-4012, tenant_id=1001, with_snapshot=false) [2024-06-25 17:00:21.662755] WDIAG [SERVER] do_batch_update_ (ob_tablet_table_updater.cpp:619) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=3][errcode=-4012] fail to start transaction(ret=-4012, ret="OB_TIMEOUT", tenant_id=1002, meta_tenant_id=1001) [2024-06-25 17:00:21.667797] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=19, this=0x2b3a9537b470, extra_info="") [2024-06-25 17:00:21.667808] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=9, avg_log_batch_cnt=1, total_group_log_size=1098, avg_group_log_size=122) [2024-06-25 17:00:21.667814] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT APPEND DATA SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1098) [2024-06-25 17:00:21.667820] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT APPEND COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a9534ee50, extra_info="id:1006") [2024-06-25 17:00:21.667823] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.667827] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.667829] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.667831] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:21.667843] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:21.667866] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=71, this=0x2b3a953853b0, extra_info="palf_id:1006") [2024-06-25 17:00:21.667880] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=1098) [2024-06-25 17:00:21.667890] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=5, this=0x2b3a953855e0, extra_info="palf_id:1006") [2024-06-25 17:00:21.668051] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-000616432547F05A-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301552, send_timestamp=1719305999366498) [2024-06-25 17:00:21.668103] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F05A-0-0] [lt=5] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a9537b6a0, extra_info="palf_id:1006") [2024-06-25 17:00:21.668111] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F05A-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=298, this=0x2b3a9537b8d0, extra_info="palf_id:1006") [2024-06-25 17:00:21.668115] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F05A-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=24, avg_log_submit_to_flush_time=86, avg_log_submit_to_slide_time=273) [2024-06-25 17:00:21.669622] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.670300] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=12064524, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=149, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:21.678566] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62243][pnio1][T1001][YB420ABF4533-00061661718F654B-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.679980] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [99154][T1004_Occam][T1004][YB420ABF4533-000616615EAF73AE-0-0] [lt=6] for each ls(succ_num=5, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:21.680050] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [99154][T1004_Occam][T1004][YB420ABF4533-000616615EAF73B0-0-0] [lt=9] for each ls(succ_num=5, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:21.681376] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1023606, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=221, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:21.683736] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.683756] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e68a080, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221537959, assoc_session_id:3221537959, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306021413788, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3a97f75ff8, id:3221537959, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e68a080}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:21.686620] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:120) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8] tenant info is invalid(ret=-4076, query_ts=0, standby_scn={val:0, v:0}, this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"10.191.69.51:2882", switch_to_leader_ts:-1}) [2024-06-25 17:00:21.686633] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:134) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=12] ObStandbyTimestampService state(*this={inited:true, last_id:-1, tenant_id:1002, epoch:-1, self:"10.191.69.51:2882", switch_to_leader_ts:-1}) [2024-06-25 17:00:21.686638] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.687353] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.702644] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=3] [PALF_DUMP](palf_id=1008, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1008, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:95833428, max_lsn:74143920356, max_scn:{val:1719306021546830000, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:74143920356}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:74101977316}}, last_submit_lsn:{lsn:74143920234}, last_submit_end_lsn:{lsn:74143920356}, last_submit_log_id:95833428, last_submit_log_pid:1, max_flushed_lsn:{lsn:74143920234}, max_flushed_end_lsn:{lsn:74143920356}, max_flushed_log_pid:1, committed_end_lsn:{lsn:74143920356}, last_slide_log_id:95833428, last_slide_scn:{val:1719306021546830000, v:0}, last_slide_lsn:{lsn:74143920234}, last_slide_end_lsn:{lsn:74143920356}, last_slide_log_pid:1, last_slide_log_accum_checksum:2006839162, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2b3a86de6530}, [StateMgr]={this:0x2b3a86dea2b0, self:"10.191.69.51:2882", palf_id:1008, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, leader:"10.186.69.171:2882", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:95833429, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x2b3a86df0b70, allow_vote:true}, [ConfigMgr]={palf_id:1008, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:1, config_seq:2}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1719306019901686, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1719306019901686, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2b3a86de7c70}, [ModeMgr]={palf_id:1008, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697228952000, v:0}}, accepted_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697228952000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697228952000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1008, is_inited:true, min_block_max_scn:{val:1718634236278240001, v:0}, min_block_id:1092, min_block_min_scn:{val:1718578883487163004, v:0}, base_lsn_for_block_gc:{lsn:74083663872}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, log_config_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, log_snapshot_meta:{version:1, base_lsn:{lsn:74083663872}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697228952000, v:0}}}, log_meta_storage:{log_tail:{lsn:2699264}, readable_log_tail:{lsn:2699264}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:74150768640}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1008}, block_mgr:{log_dir:"/data/clog/tenant_1002/1008/meta", dir_fd:821, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:64405504, block_header_serialize_buf_:0x2b3a86dec438, flashback_version:0}, log_storage:{log_tail:{lsn:74143920356}, readable_log_tail:{lsn:74143920356}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:74083663872}, min_scn:{val:1719255135031139004, v:0}, curr_block_id:1104, palf_id:1008}, block_mgr:{log_dir:"/data/clog/tenant_1002/1008/log", dir_fd:822, min_block_id:1092, max_block_id:1105, curr_writable_block_id:1104}, logical_block_size_:67104768, curr_block_writable_size_:6848284, block_header_serialize_buf_:0x2b3a86dee0b8, flashback_version:0}, palf_epoch:3, last_purge_throttling_ts:-1, this:0x2b3a86dea9f0}, [Reconfirm]={palf_id:1008, self:"10.191.69.51:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2b3a86dea430}) [2024-06-25 17:00:21.702749] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=103][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1008, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:21.702755] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF_DUMP](palf_id=1008, self="10.191.69.51:2882", ack_info_list=[]) [2024-06-25 17:00:21.702759] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1008, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:21.702764] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=4] current log_task status(palf_id=1008, self="10.191.69.51:2882", log_id=95833429, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[]) [2024-06-25 17:00:21.707023] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100186][T1004_L0_G0][T1004][YB420ABF4534-0006164514B897B3-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:21.711675] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99946][T1004_L0_G0][T1004][YB420ABF4534-0006164514B897B3-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:21.721220] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.724720] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=76000, remain_us=23002455, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=76, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:21.725394] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100262][T1004_L0_G0][T1004][YB420ABF4534-0006164514C6F306-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:21.727640] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1342, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:21.727879] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164323C50526-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5386, fly_ts=22301554, send_timestamp=1719305999426325) [2024-06-25 17:00:21.729197] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100064][T1004_L0_G0][T1004][YB420ABF4534-0006164514C6F306-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:21.731117] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=1] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=22, this=0x2bcde0525470, extra_info="") [2024-06-25 17:00:21.731132] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT APPEND COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0x2b589fc96e50, extra_info="id:1039") [2024-06-25 17:00:21.731137] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:21.731166] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:21.731170] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.731173] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.731175] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.731212] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=70, this=0x2bcde052f3b0, extra_info="palf_id:1039") [2024-06-25 17:00:21.731224] INFO [LIB] stat (utility.h:1214) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=23, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbff3b0, extra_info="") [2024-06-25 17:00:21.731236] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=4, this=0x2bcde052f5e0, extra_info="palf_id:1039") [2024-06-25 17:00:21.731328] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325962B46-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301546, send_timestamp=1719305999429781) [2024-06-25 17:00:21.731370] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B46-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=2, this=0x2bcde05256a0, extra_info="palf_id:1039") [2024-06-25 17:00:21.731378] INFO [LIB] stat (utility.h:1197) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B46-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=303, this=0x2bcde05258d0, extra_info="palf_id:1039") [2024-06-25 17:00:21.731384] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-0006164325962B47-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301547, send_timestamp=1719305999429836) [2024-06-25 17:00:21.733839] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:21.738620] INFO [COMMON] replace_fragment_node (ob_kvcache_map.cpp:696) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] Cache replace map node details(ret=0, replace_node_count=0, replace_time=0, replace_start_pos=60000000, replace_num=100000) [2024-06-25 17:00:21.738633] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=11] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=10) [2024-06-25 17:00:21.739504] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.742177] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [62581][T1_Occam][T1][YB420ABF4533-000616615CDF8FAA-0-0] [lt=9] for each ls(succ_num=1, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:21.742241] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [62581][T1_Occam][T1][YB420ABF4533-000616615CDF8FAC-0-0] [lt=8] for each ls(succ_num=1, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:21.748010] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=3][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:21.748020] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:21.756256] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061644F08BF869-0-0] [lt=6] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1006, child={server:"10.186.69.176:2882", region:"sys_region", register_time_us:1719306021756163, keepalive_ts:-1}) [2024-06-25 17:00:21.756269] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061644F08BF869-0-0] [lt=13] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1006, self="10.191.69.51:2882", child={server:"10.186.69.176:2882", region:"sys_region", register_time_us:1719306021756163, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:21.759250] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=97000, remain_us=20065728, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=97, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:21.759623] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [62160][ServerGTimer][T1004][Y0-0000000000000000-0-0] [lt=10] get_disk_usage(ret=0, capacity(MB):=2073600, used(MB):=1658818) [2024-06-25 17:00:21.761415] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325C3013B-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301552, send_timestamp=1719305999459862) [2024-06-25 17:00:21.761709] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABF4538-00061669548FCC5B-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301550, send_timestamp=1719305999460158) [2024-06-25 17:00:21.767857] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.767864] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.767867] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.767869] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:21.767871] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:21.768084] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-000616432547F05E-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301558, send_timestamp=1719305999466525) [2024-06-25 17:00:21.771307] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [99194][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=6] ====== tenant freeze timer task ====== [2024-06-25 17:00:21.776703] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=7172564, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=181, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:21.777847] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=2867493, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=209, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:21.778612] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62249][pnio1][T1001][YB420ABF4533-00061661718F654C-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.786678] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.786909] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.787096] WDIAG [RPC] send (ob_poc_rpc_proxy.h:156) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4012] sync rpc execute fail(ret=-4012, addr="10.191.69.52:2882", pcode=1315) [2024-06-25 17:00:21.787110] WDIAG [SQL.ENG] query_statistics (ob_px_tenant_target_monitor.cpp:275) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4012] send rpc to query statistics failed, need send refresh all(ret=-4012) [2024-06-25 17:00:21.787116] WDIAG [SQL.ENG] refresh_statistics (ob_px_tenant_target_monitor.cpp:114) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] query statistics failed(ret=-4012) [2024-06-25 17:00:21.787121] WDIAG [COMMON] operator() (ob_px_target_mgr.h:158) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] target monitor refresh statistics failed(ret=-4012, px_tenant_info={v:1002}, px_res_info->get_target_monitor()={is_init:true, tenant_id:1002, server:"10.191.69.51:2882", dummy_cache_leader:"10.191.69.52:2882", role:2}) [2024-06-25 17:00:21.794720] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301556, send_timestamp=1719305999493164) [2024-06-25 17:00:21.795453] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.798301] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.798447] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63430][T1001_FreInfoRe][T1001][YB420ABF4533-000616614D3FE4AB-0-0] [lt=3][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.799881] INFO [STORAGE.TABLELOCK] garbage_collect_for_all_ls_ (ob_table_lock_service.cpp:213) [99278][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=3] start to check and clear obj lock(ls->get_ls_id()={id:1039}) [2024-06-25 17:00:21.799952] INFO [STORAGE.TABLELOCK] garbage_collect_for_all_ls_ (ob_table_lock_service.cpp:213) [99278][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=8] start to check and clear obj lock(ls->get_ls_id()={id:1005}) [2024-06-25 17:00:21.800024] INFO [STORAGE.TABLELOCK] garbage_collect_for_all_ls_ (ob_table_lock_service.cpp:213) [99278][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=4] start to check and clear obj lock(ls->get_ls_id()={id:1006}) [2024-06-25 17:00:21.800059] INFO [STORAGE.TABLELOCK] garbage_collect_for_all_ls_ (ob_table_lock_service.cpp:213) [99278][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=3] start to check and clear obj lock(ls->get_ls_id()={id:1007}) [2024-06-25 17:00:21.800094] INFO [STORAGE.TABLELOCK] garbage_collect_for_all_ls_ (ob_table_lock_service.cpp:213) [99278][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=4] start to check and clear obj lock(ls->get_ls_id()={id:1008}) [2024-06-25 17:00:21.802058] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.805203] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [100469][T1003_ObTimer][T1003][Y0-0000000000000000-0-0] [lt=5] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-06-25 17:00:21.805215] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:230) [100469][T1003_ObTimer][T1003][Y0-0000000000000000-0-0] [lt=8] [tabletchange] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=2364) [2024-06-25 17:00:21.805555] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC0A4-0-0] [lt=10] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a5d1db470, extra_info="") [2024-06-25 17:00:21.805644] INFO [LIB] stat (utility.h:1197) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=12] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=73, this=0x2b3a5d1e53b0, extra_info="palf_id:1") [2024-06-25 17:00:21.805657] INFO [LIB] stat (utility.h:1214) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=10] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=6, this=0x2b3a40ffddb0, extra_info="") [2024-06-25 17:00:21.805671] INFO [LIB] stat (utility.h:1197) [62591][T1_LogIOCb0][T1][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=10, this=0x2b3a5d1e55e0, extra_info="palf_id:1") [2024-06-25 17:00:21.806820] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [99395][T1004_Flush][T1004][Y0-0000000000000000-0-0] [lt=4] ====== traversal_flush timer task ====== [2024-06-25 17:00:21.806857] INFO [MDS] flush (mds_table_mgr.cpp:172) [99395][T1004_Flush][T1004][Y0-0000000000000000-0-0] [lt=9] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1039}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x2b589fc92510, is_inited:true, freezing_scn:{val:0, v:0}, ls:{ls_meta:{tenant_id:1004, ls_id:{id:1039}, ls_create_status:1, clog_checkpoint_scn:{val:1719306017459730000, v:0}, clog_base_lsn:{lsn:2818400256}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306018784915000, v:0}, tablet_change_checkpoint_scn:{val:1719301157002134002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:1, proposal_id:1, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1039}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1039, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:50335808, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:21.806915] INFO [MDS] flush (mds_table_mgr.cpp:172) [99395][T1004_Flush][T1004][Y0-0000000000000000-0-0] [lt=41] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1005}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x2b3b5a880510, is_inited:true, freezing_scn:{val:1718180641483667000, v:0}, ls:{ls_meta:{tenant_id:1004, ls_id:{id:1005}, ls_create_status:1, clog_checkpoint_scn:{val:1718725371184341000, v:0}, clog_base_lsn:{lsn:5648141217792}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306018784915000, v:0}, tablet_change_checkpoint_scn:{val:1719301157002134002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716899681409542026, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:1, proposal_id:3, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1005}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1005, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:21.806953] INFO [MDS] flush (mds_table_mgr.cpp:172) [99395][T1004_Flush][T1004][Y0-0000000000000000-0-0] [lt=23] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1006}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x2b3b61c2c510, is_inited:true, freezing_scn:{val:1718738317058375001, v:0}, ls:{ls_meta:{tenant_id:1004, ls_id:{id:1006}, ls_create_status:1, clog_checkpoint_scn:{val:1718846540868570004, v:0}, clog_base_lsn:{lsn:8904332980224}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306018784915000, v:0}, tablet_change_checkpoint_scn:{val:1719301161935089002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716928614029017002, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1006}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1006, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:21.806988] INFO [MDS] flush (mds_table_mgr.cpp:172) [99395][T1004_Flush][T1004][Y0-0000000000000000-0-0] [lt=20] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1007}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x2b3b5b798510, is_inited:true, freezing_scn:{val:1718625962894074001, v:0}, ls:{ls_meta:{tenant_id:1004, ls_id:{id:1007}, ls_create_status:1, clog_checkpoint_scn:{val:1718726419050442000, v:0}, clog_base_lsn:{lsn:5781478391808}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306018784915000, v:0}, tablet_change_checkpoint_scn:{val:1719301167059506001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716901297164071002, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1007}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1007, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:21.807023] INFO [MDS] flush (mds_table_mgr.cpp:172) [99395][T1004_Flush][T1004][Y0-0000000000000000-0-0] [lt=20] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1008}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x2b3b5b6a4510, is_inited:true, freezing_scn:{val:1718844273612110000, v:0}, ls:{ls_meta:{tenant_id:1004, ls_id:{id:1008}, ls_create_status:1, clog_checkpoint_scn:{val:1719196007356667000, v:0}, clog_base_lsn:{lsn:5845831864320}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306018784915000, v:0}, tablet_change_checkpoint_scn:{val:1719301171837486004, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716901298196291003, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1008}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1008, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:7936, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:21.807043] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:336) [99395][T1004_Flush][T1004][Y0-0000000000000000-0-0] [lt=19] succeed to traversal_flush(ret=0, ls_cnt=5) [2024-06-25 17:00:21.807600] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=9] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289109524480, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:21.807799] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.812209] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63252][T1002_FreInfoRe][T1001][YB420ABF4533-000616616D9E8643-0-0] [lt=3][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.817555] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.821018] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62744][T1_ReplaySrv1][T1][Y0-0000000000000000-0-0] [lt=4] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:21.821454] INFO do_work (ob_rl_mgr.cpp:709) [62262][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=7] swc wakeup.(stat_period_=1000000, ready=false) [2024-06-25 17:00:21.824139] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [63432][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=0] run rewrite rule refresh task(rule_mgr_->tenant_id_=1001) [2024-06-25 17:00:21.824164] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=11910660, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=150, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:21.824813] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63432][T1001_ReqMemEvi][T1001][YB420ABF4533-000616613D6F1175-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:21.826623] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95434][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB4A-0-0] [lt=2] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:21.826847] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC909-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:21.830932] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.831146] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:21.831179] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:21.831184] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.831186] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.831189] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.831238] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1039, self="10.191.69.51:2882", committed size=1332) [2024-06-25 17:00:21.831378] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABDC060-0006164325962B4B-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301558, send_timestamp=1719305999529819) [2024-06-25 17:00:21.831420] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-0006164325962B4C-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301547, send_timestamp=1719305999529873) [2024-06-25 17:00:21.831424] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B4B-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1039, self="10.191.69.51:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=27, avg_log_submit_to_flush_time=89, avg_log_submit_to_slide_time=278) [2024-06-25 17:00:21.831459] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B4C-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1005, self="10.191.69.51:2882", total_slide_log_cnt=14, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=21, avg_log_submit_to_flush_time=94, avg_log_submit_to_slide_time=271) [2024-06-25 17:00:21.833788] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.833893] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:21.834770] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=16][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4013, dropped:291, tid:65259}]) [2024-06-25 17:00:21.835081] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=869900, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=222, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:21.837612] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.837621] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.837626] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.837632] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.837637] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.839558] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.840691] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABF4533-0006166172BEBEC2-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301550, send_timestamp=1719305999539140) [2024-06-25 17:00:21.843497] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.843505] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.843510] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.843515] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.843520] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.853305] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.853313] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:21.853316] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.853322] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.853326] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.853778] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [63295][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=4] ====== traversal_flush timer task ====== [2024-06-25 17:00:21.853817] INFO [MDS] flush (mds_table_mgr.cpp:172) [63295][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=8] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1006}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x2b3a9534a510, is_inited:true, freezing_scn:{val:1718285174571744002, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1006}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018752321001, v:0}, clog_base_lsn:{lsn:75425759232}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315392841000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:1, proposal_id:1, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1006}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1006, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:110, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:21.853869] INFO [MDS] flush (mds_table_mgr.cpp:172) [63295][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=34] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1007}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x2b3a86ce4510, is_inited:true, freezing_scn:{val:1718289975940695002, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1007}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018870127000, v:0}, clog_base_lsn:{lsn:85155950592}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315408583001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1007}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1007, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:21.853905] INFO [MDS] flush (mds_table_mgr.cpp:172) [63295][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=21] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1008}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x2b3a86daa510, is_inited:true, freezing_scn:{val:1718286006014126001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1008}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018543922003, v:0}, clog_base_lsn:{lsn:74083663872}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315055094004, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1008}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1008, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-550728704, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:21.853939] INFO [MDS] flush (mds_table_mgr.cpp:172) [63295][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=19] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1009}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x2b3aaf086510, is_inited:true, freezing_scn:{val:1718283438009682001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1009}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018805948000, v:0}, clog_base_lsn:{lsn:86632255488}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315224596003, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1009}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1009, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:21.853973] INFO [MDS] flush (mds_table_mgr.cpp:172) [63295][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=19] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1010}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x2b3aaf132510, is_inited:true, freezing_scn:{val:1718283437110618003, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1010}, ls_create_status:1, clog_checkpoint_scn:{val:1719198894656674000, v:0}, clog_base_lsn:{lsn:75627073536}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315483366000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1010}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1010, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:21.853992] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:336) [63295][T1002_Flush][T1002][Y0-0000000000000000-0-0] [lt=18] succeed to traversal_flush(ret=0, ls_cnt=5) [2024-06-25 17:00:21.854645] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=77000, remain_us=22872530, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=77, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:21.857412] WDIAG [PALF] handle_register_parent_req (log_config_mgr.cpp:2789) [95434][T1004_L0_G1][T1004][YB420ABDC061-00061643AA65BDF6-0-0] [lt=4][errcode=-4002] registering child is not in learner list or is migrating(palf_id=1039, self="10.191.69.51:2882", child={server:"10.189.192.97:2882", region:"sys_region", register_time_us:1719305971236691, keepalive_ts:-1}, in_list_ret=-4018, learner_in_list={server:"0.0.0.0:0", timestamp:-1, flag:0}) [2024-06-25 17:00:21.857425] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95434][T1004_L0_G1][T1004][YB420ABDC061-00061643AA65BDF6-0-0] [lt=11] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1039, self="10.191.69.51:2882", child={server:"10.189.192.97:2882", region:"sys_region", register_time_us:1719305971236691, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:21.861744] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABF4538-00061669548FCC5F-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301557, send_timestamp=1719305999560186) [2024-06-25 17:00:21.861766] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABF4538-00061669548FCC5F-0-0] [lt=9] [PALF STAT LOG TASK TIME](palf_id=1010, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=94, avg_log_submit_to_slide_time=281) [2024-06-25 17:00:21.861770] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=13] [PALF STAT HOT CACHE HIT RATE](palf_id=1010, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:21.863569] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1B99-0-0] [lt=10] [PALF STAT COMMITTED LOG SIZE](palf_id=1009, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:21.863639] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:21.867415] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.867424] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.867429] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.867436] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.867440] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.867908] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.867914] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.867916] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.867919] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:21.867921] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:21.868134] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-000616432547F062-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301551, send_timestamp=1719305999566582) [2024-06-25 17:00:21.871163] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.871172] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.871177] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.871182] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.871187] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.876803] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164324713063-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=22301549, send_timestamp=1719305999575253) [2024-06-25 17:00:21.878898] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62243][pnio1][T1001][YB420ABF4533-00061661718F654D-0-0] [lt=12][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:21.880379] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=2][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:21.880389] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:21.880395] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=4][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999958, timeout_timestamp=1719306051879982) [2024-06-25 17:00:21.880403] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.880405] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:21.880409] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:21.880414] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:21.880417] WDIAG [SQL] open (ob_result_set.cpp:157) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=3][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:21.880420] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:21.880424] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=4][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}) [2024-06-25 17:00:21.880430] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=5][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.880436] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=6][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:21.880445] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=7][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:21.880448] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=3][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:21.880452] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=4][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:21.880457] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85A-0-0] [lt=3][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:21.880463] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port"}, aret=-4013, ret=-4013) [2024-06-25 17:00:21.880466] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:21.880472] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=5][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:21.880475] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=2][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:21.880478] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=3][errcode=-4013] query failed(ret=-4013, conn=0x2b6272f18eb0, start=1719306021879982, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:21.880484] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=6][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:21.880488] WDIAG [SHARE.PT] get_by_tenant (ob_persistent_ls_table.cpp:639) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, sql=SELECT * FROM __all_ls_meta_table WHERE tenant_id = 1002 ORDER BY tenant_id, ls_id, svr_ip, svr_port) [2024-06-25 17:00:21.880512] WDIAG [SHARE.PT] get_by_tenant (ob_ls_table_operator.cpp:252) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=5][errcode=-4013] get all ls info by persistent_ls_ failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002) [2024-06-25 17:00:21.880517] WDIAG [SHARE] inner_open_ (ob_ls_table_iterator.cpp:104) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=4][errcode=-4013] fail to get ls infos by tenant(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, inner_table_only=false) [2024-06-25 17:00:21.880520] WDIAG [SHARE] next (ob_ls_table_iterator.cpp:71) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=3][errcode=-4013] fail to open iterator(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.880525] WDIAG [SERVER] build_replica_map_ (ob_tenant_meta_checker.cpp:334) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=5][errcode=-4013] ls table iterator next failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.880529] WDIAG [SERVER] check_ls_table_ (ob_tenant_meta_checker.cpp:214) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=3][errcode=-4013] build replica map from ls table failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", mode=0) [2024-06-25 17:00:21.880535] WDIAG [SERVER] check_ls_table (ob_tenant_meta_checker.cpp:188) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=5][errcode=-4013] check ls table failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", mode=0) [2024-06-25 17:00:21.880538] WDIAG [SERVER] runTimerTask (ob_tenant_meta_checker.cpp:44) [63280][T1002_LSMetaCh][T1002][YB420ABF4533-000616615C0FE85A-0-0] [lt=3][errcode=-4013] fail to check ls meta table(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.880874] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABF4533-0006166172BEBEC7-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301545, send_timestamp=1719305999579328) [2024-06-25 17:00:21.886735] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.894474] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100065][T1004_L0_G0][T1004][YB420ABF4538-00061669288F87EE-0-0] [lt=4][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:21.898997] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100187][T1004_L0_G0][T1004][YB420ABF4538-00061669288F87EE-0-0] [lt=8][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:21.902761] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=10] re_register_parent reason(palf_id=1010, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306017900598, last_first_register_time_us=1713323717486317, register_time_us=1719306017900598, curr_time_us=1719306021902761) [2024-06-25 17:00:21.902784] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=9] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1010, self="10.191.69.51:2882", reason=1, leader="10.189.192.96:2882", register_time_us=1719306021902771, last_submit_register_req_time_us=1719306021902771, register_parent_reason=1) [2024-06-25 17:00:21.902791] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=6] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1010, self="10.191.69.51:2882") [2024-06-25 17:00:21.905025] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.905033] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:21.905038] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:21.905043] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.905048] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.906761] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [100348][T1003_LogLoop][T1003][Y0-0000000000000000-0-0] [lt=6] LogLoopThread round_cost_time(us)(round_cost_time=0) [2024-06-25 17:00:21.910020] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=98000, remain_us=19914958, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=98, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:21.910623] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [62721][T1_Flush][T1][Y0-0000000000000000-0-0] [lt=4] ====== traversal_flush timer task ====== [2024-06-25 17:00:21.910688] INFO [MDS] flush (mds_table_mgr.cpp:172) [62721][T1_Flush][T1][Y0-0000000000000000-0-0] [lt=8] [FREEZE]no valid mds table there, no need do flush(ret=0, ret="OB_SUCCESS", ls_->get_ls_id()={id:1}, recycle_scn={val:4611686018427387903, v:0}, need_freeze=false, order_flusher_for_some={high_priority_flusher:size:0, ordered_tablets:{}, max_consequent_callbacked_scn={val:18446744073709551615, v:3}, *this={this:0x2b3a5d156510, is_inited:true, freezing_scn:{val:1719272886793317000, v:0}, ls:{ls_meta:{tenant_id:1, ls_id:{id:1}, ls_create_status:1, clog_checkpoint_scn:{val:1719272888853692000, v:0}, clog_base_lsn:{lsn:35095793664}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:1719285609440317000, v:0}, all_id_meta:{id_meta:[{limited_id:1719306035353935000, latest_log_ts:{val:1719306018786724001, v:0}}, {limited_id:80000001, latest_log_ts:{val:1719298896011036001, v:0}}, {limited_id:2660000001, latest_log_ts:{val:1719295281004326001, v:0}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a437f0030, is_in_stop_state_:false, is_inited_:true, id_:1}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:21.910720] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:336) [62721][T1_Flush][T1][Y0-0000000000000000-0-0] [lt=30] succeed to traversal_flush(ret=0, ls_cnt=1) [2024-06-25 17:00:21.911185] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=3] [PALF_DUMP](palf_id=1005, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1005, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:448028956, max_lsn:5678775313633, max_scn:{val:1719306021463662003, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:5678775313633}, reserved_buffer_size:41943040, available_buffer_size:33554432, readable_begin_lsn:{lsn:5678733370593}}, last_submit_lsn:{lsn:5678775313511}, last_submit_end_lsn:{lsn:5678775313633}, last_submit_log_id:448028956, last_submit_log_pid:3, max_flushed_lsn:{lsn:5678775313511}, max_flushed_end_lsn:{lsn:5678775313633}, max_flushed_log_pid:3, committed_end_lsn:{lsn:5678775313633}, last_slide_log_id:448028956, last_slide_scn:{val:1719306021463662003, v:0}, last_slide_lsn:{lsn:5678775313511}, last_slide_end_lsn:{lsn:5678775313633}, last_slide_log_pid:3, last_slide_log_accum_checksum:1185781943, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2b3b5a816530}, [StateMgr]={this:0x2b3b5a81a2b0, self:"10.191.69.51:2882", palf_id:1005, role:"LEADER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:3}, leader:"10.191.69.51:2882", leader_epoch:2, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:1745558943762}, scan_disk_log_finished:true, last_check_start_id:448028957, is_changing_config_with_arb:false, reconfirm_start_time_us:1716561397886359, palf_role_change_cb:0x2b3b5a820b70, allow_vote:true}, [ConfigMgr]={palf_id:1005, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:3, prev:{version:1, config:{config_version:{proposal_id:3, config_seq:145}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:17196, lock_time:1716899681388790}}, curr:{version:1, config:{config_version:{proposal_id:3, config_seq:146}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17196, lock_time:1716899681846513}}, prev_log_proposal_id:3, prev_lsn:{lsn:3128521627842}, prev_mode_pid:3}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:17196, lock_type:0, added_list:0, removed_list:0, type:"UNLOCK_CONFIG_CHANGE"}, state:0, checking_barrier:{prev_log_proposal_id:3, prev_lsn:{lsn:3128521627842}, prev_end_lsn:{lsn:3128521634694}, prev_mode_pid:3}, reconfig_barrier:{prev_log_proposal_id:3, prev_lsn:{lsn:3128521627842}, prev_end_lsn:{lsn:3128521634694}, prev_mode_pid:3}, persistent_config_version:{proposal_id:3, config_seq:146}, ms_ack_list:0, resend_config_version:{proposal_id:3, config_seq:146}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1716561397486120, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1716561397486120, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2b3b5a817c70}, [ModeMgr]={palf_id:1005, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:3, mode_version:0, access_mode:1, ref_scn:{val:1713332543407963001, v:0}}, accepted_mode_meta:{version:1, proposal_id:3, mode_version:0, access_mode:1, ref_scn:{val:1713332543407963001, v:0}}, last_submit_mode_meta:{version:1, proposal_id:3, mode_version:0, access_mode:1, ref_scn:{val:1713332543407963001, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:1716561398086523, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1005, is_inited:true, min_block_max_scn:{val:1718674656277454003, v:0}, min_block_id:84137, min_block_min_scn:{val:1718671656999387017, v:0}, base_lsn_for_block_gc:{lsn:5648141217792}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:3}, log_config_meta:{version:3, proposal_id:3, prev:{version:1, config:{config_version:{proposal_id:3, config_seq:145}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:17196, lock_time:1716899681388790}}, curr:{version:1, config:{config_version:{proposal_id:3, config_seq:146}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17196, lock_time:1716899681846513}}, prev_log_proposal_id:3, prev_lsn:{lsn:3128521627842}, prev_mode_pid:3}, log_snapshot_meta:{version:1, base_lsn:{lsn:5648141217792}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:3, mode_version:0, access_mode:1, ref_scn:{val:1713332543407963001, v:0}}}, log_meta_storage:{log_tail:{lsn:1679360}, readable_log_tail:{lsn:1679360}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:5678808096768}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1005}, block_mgr:{log_dir:"/data/clog/tenant_1004/1005/meta", dir_fd:835, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:65425408, block_header_serialize_buf_:0x2b3b5a81c438, flashback_version:0}, log_storage:{log_tail:{lsn:5678775313633}, readable_log_tail:{lsn:5678775313633}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:5678740992000}, min_scn:{val:1719298149934200005, v:0}, curr_block_id:84625, palf_id:1005}, block_mgr:{log_dir:"/data/clog/tenant_1004/1005/log", dir_fd:845, min_block_id:84137, max_block_id:84626, curr_writable_block_id:84625}, logical_block_size_:67104768, curr_block_writable_size_:32783135, block_header_serialize_buf_:0x2b3b5a81e0b8, flashback_version:0}, palf_epoch:1, last_purge_throttling_ts:1716899553213145, this:0x2b3b5a81a9f0}, [Reconfirm]={palf_id:1005, self:"10.191.69.51:2882", state:"FINISHED", new_proposal_id:3, prepare_log_ack_list:3"10.189.192.96:2882";"10.191.69.56:2882";"10.186.69.176:2882";, curr_paxos_follower_list:4{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, majority_cnt:3, majority_max_log_server:"10.191.69.51:2882", majority_max_accept_pid:2, majority_max_lsn:{lsn:1750156420456}, saved_end_lsn:{lsn:1750156420456}, last_submit_prepare_req_time_us:1716561397886487, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2b3b5a81a430}) [2024-06-25 17:00:21.911317] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=129] [PALF_DUMP](palf_id=1005, self="10.191.69.51:2882", ack_info_list=[{member:{server:"10.186.69.171:2882", timestamp:1, flag:0}, last_ack_time_us:1719306021832035, last_flushed_end_lsn:{lsn:5678775313633}}, {member:{server:"10.186.69.176:2882", timestamp:1, flag:0}, last_ack_time_us:1719306021832028, last_flushed_end_lsn:{lsn:5678775313633}}, {member:{server:"10.189.192.96:2882", timestamp:1, flag:0}, last_ack_time_us:1719306021831438, last_flushed_end_lsn:{lsn:5678775313633}}, {member:{server:"10.191.69.51:2882", timestamp:1, flag:0}, last_ack_time_us:1719306021831261, last_flushed_end_lsn:{lsn:5678775313633}}, {member:{server:"10.191.69.56:2882", timestamp:1, flag:0}, last_ack_time_us:1719306021831354, last_flushed_end_lsn:{lsn:5678775313633}}]) [2024-06-25 17:00:21.911332] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=14] current log_task status(palf_id=1005, self="10.191.69.51:2882", log_id=448028957, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[{member:{server:"10.186.69.171:2882", timestamp:1, flag:0}, last_ack_time_us:1719306021832035, last_flushed_end_lsn:{lsn:5678775313633}}, {member:{server:"10.186.69.176:2882", timestamp:1, flag:0}, last_ack_time_us:1719306021832028, last_flushed_end_lsn:{lsn:5678775313633}}, {member:{server:"10.189.192.96:2882", timestamp:1, flag:0}, last_ack_time_us:1719306021831438, last_flushed_end_lsn:{lsn:5678775313633}}, {member:{server:"10.191.69.51:2882", timestamp:1, flag:0}, last_ack_time_us:1719306021831261, last_flushed_end_lsn:{lsn:5678775313633}}, {member:{server:"10.191.69.56:2882", timestamp:1, flag:0}, last_ack_time_us:1719306021831354, last_flushed_end_lsn:{lsn:5678775313633}}]) [2024-06-25 17:00:21.916768] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.916778] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:21.916783] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:21.916790] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.916795] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.919015] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [63357][T1001_LogLoop][T1001][Y0-0000000000000000-0-0] [lt=9] LogLoopThread round_cost_time(us)(round_cost_time=0) [2024-06-25 17:00:21.922871] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [62718][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=969588866, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=1, size_used=409014, mem_used=767425536) [2024-06-25 17:00:21.930442] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=7018826, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=182, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:21.931199] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:21.931238] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT GROUP LOG INFO](palf_id=1005, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=14, avg_log_batch_cnt=1, total_group_log_size=3296, avg_group_log_size=235) [2024-06-25 17:00:21.931247] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:21.931251] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.931255] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.931259] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.931409] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABDC060-0006164325962B50-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301550, send_timestamp=1719305999629858) [2024-06-25 17:00:21.931513] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=2713827, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=210, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:21.933170] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.933179] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:21.933184] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:21.933190] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.933195] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.933263] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=6][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:21.933273] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:21.933279] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=3][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999964, timeout_timestamp=1719306051932779) [2024-06-25 17:00:21.933285] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:21.933287] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:21.933291] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:21.933296] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:21.933299] WDIAG [SQL] open (ob_result_set.cpp:157) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=3][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:21.933302] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=3][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:21.933306] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=3][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT value from oceanbase.__all_sys_stat where name = 'current_timezone_version'"}) [2024-06-25 17:00:21.933311] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=5][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT value from oceanbase.__all_sys_stat where name = 'current_timezone_version'"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:21.933317] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=5][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:21.933324] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=6][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:21.933327] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=3][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:21.933329] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:21.933336] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62907][TimezoneMgr][T1001][YB420ABF4533-0006166101CF5DAB-0-0] [lt=4][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT value from oceanbase.__all_sys_stat where name = 'current_timezone_version'"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:21.933340] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62907][TimezoneMgr][T0][YB420ABF4533-0006166101CF5DAB-0-0] [lt=3][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT value from oceanbase.__all_sys_stat where name = 'current_timezone_version'"}, aret=-4013, ret=-4013) [2024-06-25 17:00:21.933345] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [62907][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT value from oceanbase.__all_sys_stat where name = 'current_timezone_version') [2024-06-25 17:00:21.933349] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62907][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:21.933352] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [62907][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:21.933357] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [62907][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] query failed(ret=-4013, conn=0x2b3b5a0828f0, start=1719306021932779, sql=SELECT value from oceanbase.__all_sys_stat where name = 'current_timezone_version') [2024-06-25 17:00:21.933361] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [62907][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:21.933366] WDIAG [SHARE] fetch_time_zone_info (ob_time_zone_info_manager.cpp:179) [62907][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] fail to execute sql(ret=-4013, tenant_id_=1001) [2024-06-25 17:00:21.933381] WDIAG [SERVER.OMT] operator() (ob_tenant_timezone_mgr.cpp:56) [62907][TimezoneMgr][T0][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] fail to update time zone info(ret=-4013) [2024-06-25 17:00:21.933949] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:21.938701] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=5] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=9) [2024-06-25 17:00:21.939612] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.949860] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [100333][T1003_Occam][T1003][YB420ABF4533-000616615EBF73AE-0-0] [lt=7] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:21.949876] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [100333][T1003_Occam][T1003][YB420ABF4533-000616615EBF73B0-0-0] [lt=11] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:21.950142] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:21.955654] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [42764][T1002_ComTimerT][T1002][Y0-0000000000000000-0-0] [lt=9] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-06-25 17:00:21.957137] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B53-0-0] [lt=6] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=86, avg_log_submit_to_slide_time=830) [2024-06-25 17:00:21.957144] INFO [PALF] read (log_cache.cpp:100) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:21.961523] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-0006164325C30143-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301553, send_timestamp=1719305999659970) [2024-06-25 17:00:21.961545] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C30143-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf16f470, extra_info="") [2024-06-25 17:00:21.961634] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=80, this=0x2b3aaf1793b0, extra_info="palf_id:1010") [2024-06-25 17:00:21.961645] INFO [LIB] stat (utility.h:1214) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=5, this=0x2b3a8ce1d3b0, extra_info="") [2024-06-25 17:00:21.961654] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=6, this=0x2b3aaf1795e0, extra_info="palf_id:1010") [2024-06-25 17:00:21.961821] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABF4538-00061669548FCC63-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301548, send_timestamp=1719305999660272) [2024-06-25 17:00:21.961841] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4538-00061669548FCC63-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf16f6a0, extra_info="palf_id:1010") [2024-06-25 17:00:21.961846] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4538-00061669548FCC63-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=282, this=0x2b3aaf16f8d0, extra_info="palf_id:1010") [2024-06-25 17:00:21.963042] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=12] LogLoopThread round_cost_time(us)(round_cost_time=6) [2024-06-25 17:00:21.963772] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=8] gts statistics(tenant_id=1001, gts_rpc_cnt=10, get_gts_cache_cnt=100, get_gts_with_stc_cnt=9, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-06-25 17:00:21.964084] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62248][pnio1][T0][YB420ABF4533-0006166171AF65F2-0-0] [lt=11][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:21.964094] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62248][pnio1][T0][YB420ABF4533-0006166171AF65F2-0-0] [lt=9] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:21.964106] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62248][pnio1][T0][YB420ABF4533-0006166171AF65F2-0-0] [lt=5] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306021963767]}) [2024-06-25 17:00:21.967975] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:21.967983] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:21.967986] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:21.967988] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:21.967991] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:21.968202] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-000616432547F066-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301548, send_timestamp=1719305999666653) [2024-06-25 17:00:21.970899] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:21.970909] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:21.970914] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:21.970919] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:21.970925] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:21.978031] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=11756792, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=151, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:21.978382] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [63468][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=9][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-06-25 17:00:21.978390] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [63468][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=7] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-06-25 17:00:21.978719] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62245][pnio1][T1001][YB420ABF4533-00061661718F654E-0-0] [lt=9][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:21.985500] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=78000, remain_us=22741676, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=78, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:21.986791] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:21.988789] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=716192, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=223, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:22.005656] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC0A8-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=3875, avg_group_log_size=387) [2024-06-25 17:00:22.005742] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=5] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=4414) [2024-06-25 17:00:22.006490] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A09E-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a5d1db6a0, extra_info="palf_id:1") [2024-06-25 17:00:22.006497] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A09E-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=831, this=0x2b3a5d1db8d0, extra_info="palf_id:1") [2024-06-25 17:00:22.007817] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=11] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289109524480, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:22.011234] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=18] LogLoopThread round_cost_time(us)(round_cost_time=184) [2024-06-25 17:00:22.023701] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.023722] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=20][errcode=-4013] [2024-06-25 17:00:22.023727] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.023734] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.023739] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.026563] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.026572] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.026576] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:22.026585] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.026590] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.027571] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63474][T1001_LSArchive][T1001][YB420ABF4533-000616423F8E6725-0-0] [lt=8] gc stale ls task succ [2024-06-25 17:00:22.027578] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95434][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B54-0-0] [lt=11] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=94, avg_log_submit_to_slide_time=844) [2024-06-25 17:00:22.027584] INFO [PALF] read (log_cache.cpp:100) [99266][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=14] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:22.027865] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100250][T1004_L0_G0][T1004][YB420ABF4534-000616450A3F3687-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:22.030400] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.030407] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:22.030412] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:22.030418] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.030422] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.031161] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=24] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:22.031257] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:22.031290] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:22.031294] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.031297] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.031300] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.031508] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-0006164325962B55-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301552, send_timestamp=1719305999729955) [2024-06-25 17:00:22.031527] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABDC060-0006164325962B56-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301554, send_timestamp=1719305999729972) [2024-06-25 17:00:22.032091] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100258][T1004_L0_G0][T1004][YB420ABF4534-000616450A3F3687-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:22.033532] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95434][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF5A37-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=13, avg_log_batch_cnt=0, total_group_log_size=38668, avg_group_log_size=2974) [2024-06-25 17:00:22.033898] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [99170][T1004_PalfGC][T1004][Y0-0000000000000000-0-0] [lt=12] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):2073600, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):2073600, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):500452, sequence:1}) [2024-06-25 17:00:22.034001] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:22.036251] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.036260] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.036265] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:22.036271] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.036275] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.039669] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.046079] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.046088] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.046092] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.046098] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.046102] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.051501] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62749][T1_ReplaySrv6][T1][Y0-0000000000000000-0-0] [lt=10] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:22.053053] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [99392][T1004_ReqMemEvi][T1004][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1004, evict_high_mem_level=20669530113, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=8345619, mem_used=15148855296) [2024-06-25 17:00:22.053682] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63292][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=7] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7297, mem_used=12478464) [2024-06-25 17:00:22.056429] INFO [LIB] stat (utility.h:1214) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=23, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbfec70, extra_info="") [2024-06-25 17:00:22.061576] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164325C30147-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301552, send_timestamp=1719305999760023) [2024-06-25 17:00:22.061634] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=1] will sleep(sleep_us=99000, remain_us=19763345, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=99, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:22.061847] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABF4533-00061661728FC94E-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301552, send_timestamp=1719305999760295) [2024-06-25 17:00:22.062929] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.062938] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.062944] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.062950] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.062955] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.062968] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e698d80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221701033, assoc_session_id:3221701033, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306021794060, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3aee2161f8, id:3221701033, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e698d80}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:22.063671] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1BA1-0-0] [lt=3] [PALF STAT GROUP LOG INFO](palf_id=1009, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:22.064536] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F069-0-0] [lt=11] [PALF STAT LOG TASK TIME](palf_id=1009, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=77, avg_log_submit_to_slide_time=845) [2024-06-25 17:00:22.064544] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT HOT CACHE HIT RATE](palf_id=1009, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:22.068028] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.068035] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.068037] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.068040] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:22.068042] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:22.078767] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62234][pnio1][T1001][YB420ABF4533-00061661718F654F-0-0] [lt=10][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:22.084109] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=6865159, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=183, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:22.085110] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=30][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.085128] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:103) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=1] already timeout, do not need sleep(sleep_us=0, remain_us=29946307, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=0, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.085162] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2560178, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=211, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:22.086846] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.094037] INFO [PALF] runTimerTask (block_gc_timer_task.cpp:101) [99170][T1004_PalfGC][T1004][Y0-0000000000000000-0-0] [lt=14] BlockGCTimerTask success(ret=0, cost_time_us=4, palf_env_impl_={IPalfEnvImpl:{IPalfEnvImpl:"Dummy"}, self:"10.191.69.51:2882", log_dir:"/data/clog/tenant_1004", disk_options_wrapper:{disk_opts_for_stopping_writing:{log_disk_size(MB):2073600, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):2073600, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):500452, sequence:1}, log_alloc_mgr_:{flying_log_task:0, flying_meta_task:0}}) [2024-06-25 17:00:22.095798] INFO [SERVER.OMT] check_worker_count (ob_tenant.cpp:488) [62538][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=361] worker thread began to shrink(tenant_->id()=1004, group_id_=1, token=0) [2024-06-25 17:00:22.095877] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.095885] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:22.095889] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.095895] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.095900] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.097416] INFO [SERVER.OMT] check_worker_count (ob_tenant.cpp:503) [95434][T1004_L0_G1][T1004][Y0-0000000000000000-0-0] [lt=9] worker thread exit(tenant_->id()=1004, workers_.get_size()=1) [2024-06-25 17:00:22.097427] INFO [SHARE] end_run (ob_tenant_base.cpp:332) [95434][T1004_L0_G1][T0][Y0-0000000000000000-0-0] [lt=8] tenant thread end_run(id_=1004, ret=0, thread_count_=1299) [2024-06-25 17:00:22.097435] INFO unregister_pm (ob_page_manager.cpp:50) [95434][T1004_L0_G1][T0][Y0-0000000000000000-0-0] [lt=4] unregister pm finish(&pm=0x2b5135256d80, pm.get_tid()=95434) [2024-06-25 17:00:22.100275] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=5] ====== check clog disk timer task ====== [2024-06-25 17:00:22.100287] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=6] get_disk_usage(ret=0, capacity(MB):=2073600, used(MB):=1658818) [2024-06-25 17:00:22.100299] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=4] cannot_recycle_log_size statistics(cannot_recycle_log_size=524426689176, threshold=652298158080, need_update_checkpoint_scn=false) [2024-06-25 17:00:22.100762] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.100770] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.100775] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.100780] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.100786] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.107713] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=30] new mysql sessid created(conn.sessid_=3221524510, support_ssl=false) [2024-06-25 17:00:22.107734] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=9] sm conn init succ(conn.sessid_=3221524510, sess.client_addr_="10.191.69.51:48044") [2024-06-25 17:00:22.107742] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=6] accept one succ(*s={this:0x2b70981fed70, session_id:3221524510, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:22.107936] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x2b70981fed70, session_id:3221524510, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:22.107946] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=8] kill and revert session(conn.sessid_=3221524510, proxy_sessid=0, server_id=1, ret=0) [2024-06-25 17:00:22.107950] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:918) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=4] sock ref clean, do destroy(*s={this:0x2b70981fed70, session_id:3221524510, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:22.107957] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:223) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=6] mark session id unused(conn.sessid_=3221524510) [2024-06-25 17:00:22.107961] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [62495][sql_nio16][T0][Y0-0000000000000000-0-0] [lt=3] connection close(sessid=3221524510, proxy_sessid=0, tenant_id=0, server_id=1, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=false, ret=0, trace_id=Y0-0000000000000000-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:0], disconnect_state=0) [2024-06-25 17:00:22.117387] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=79000, remain_us=22609788, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=79, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:22.124406] WDIAG [STORAGE.TRANS] process_ (ob_gti_rpc.h:178) [62243][pnio1][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=10][errcode=0] gti rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:22.126839] INFO [SHARE] get_next_sess_id (ob_active_session_guard.cpp:360) [95800][][T0][Y0-0000000000000000-0-0] [lt=0] succ to generate background session id(sessid=4449397140094976) [2024-06-25 17:00:22.126854] INFO register_pm (ob_page_manager.cpp:40) [95800][][T0][Y0-0000000000000000-0-0] [lt=9] register pm finish(ret=0, &pm=0x2bce69056d80, pm.get_tid()=95800, tenant_id=500) [2024-06-25 17:00:22.126872] INFO [SHARE] pre_run (ob_tenant_base.cpp:314) [95800][][T1004][Y0-0000000000000000-0-0] [lt=5] tenant thread pre_run(MTL_ID()=1004, ret=0, thread_count_=1300) [2024-06-25 17:00:22.126873] INFO [SERVER.OMT] recv_group_request (ob_tenant.cpp:1292) [62234][pnio1][T0][YB420ABA45AB-000616C810AFBB56-0-0] [lt=12] worker thread created(id()=1004, group->group_id_=1) [2024-06-25 17:00:22.126878] INFO [RPC.OBRPC] th_init (ob_rpc_translator.cpp:33) [95800][][T1004][Y0-0000000000000000-0-0] [lt=5] Init thread local success [2024-06-25 17:00:22.126883] INFO unregister_pm (ob_page_manager.cpp:50) [95800][][T1004][Y0-0000000000000000-0-0] [lt=4] unregister pm finish(&pm=0x2bce69056d80, pm.get_tid()=95800) [2024-06-25 17:00:22.126889] INFO register_pm (ob_page_manager.cpp:40) [95800][][T1004][Y0-0000000000000000-0-0] [lt=4] register pm finish(ret=0, &pm=0x2bce69056d80, pm.get_tid()=95800, tenant_id=1004) [2024-06-25 17:00:22.126903] INFO [OCCAM] get_idx (ob_occam_time_guard.h:224) [95800][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB56-0-0] [lt=3] init point thread id with(&point=0x563ae08112c0, idx_=1592, point=[thread id=95800, timeout ts=08:00:00.0, last click point="(null):(null):0", last click ts=08:00:00.0], thread_id=95800) [2024-06-25 17:00:22.126922] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95800][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB56-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:22.127827] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F06B-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=4, avg_log_submit_to_flush_time=90, avg_log_submit_to_slide_time=837) [2024-06-25 17:00:22.131297] INFO eloop_run (eloop.c:143) [62232][pnio1][T0][Y0-0000000000000000-0-0] [lt=7] PNIO [ratelimit] time: 1719306022131296, bytes: 29137868048790, bw: 0.627426 MB/s, add_ts: 1000456, add_bytes: 658204 [2024-06-25 17:00:22.131306] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] [PALF STAT GROUP LOG INFO](palf_id=1039, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=11, avg_log_batch_cnt=1, total_group_log_size=1342, avg_group_log_size=122) [2024-06-25 17:00:22.131317] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT APPEND DATA SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1332) [2024-06-25 17:00:22.131324] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:22.131351] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] [PALF STAT APPEND DATA SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, append size=3296) [2024-06-25 17:00:22.131356] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:22.131359] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.131362] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.131364] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.131427] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=3306) [2024-06-25 17:00:22.131551] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABDC060-0006164325962B5A-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301565, send_timestamp=1719305999829985) [2024-06-25 17:00:22.131586] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-0006164325962B5B-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301555, send_timestamp=1719305999830031) [2024-06-25 17:00:22.131897] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=11602927, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=152, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:22.133595] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1622) [2024-06-25 17:00:22.134060] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:22.138735] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=10] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=8) [2024-06-25 17:00:22.138785] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=1] will sleep(sleep_us=1000, remain_us=29892650, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=1, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.139723] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.141991] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62421][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.142300] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62419][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.142479] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62420][RpcIO][T0][Y0-0000000000000000-0-0] [lt=4] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.142487] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62424][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.142493] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=562488, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=224, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:22.142572] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62432][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.142618] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62416][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.142638] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62426][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.142665] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62417][RpcIO][T0][Y0-0000000000000000-0-0] [lt=5] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.142676] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62422][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.142828] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62423][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.142839] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62415][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.143026] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62431][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.143035] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62429][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.143061] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62418][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.143145] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62425][RpcIO][T0][Y0-0000000000000000-0-0] [lt=5] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.143282] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62430][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.143492] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62427][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.143575] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62428][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.146407] INFO [SERVER.OMT] check_worker_count (ob_tenant.cpp:488) [62538][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=10] worker thread began to shrink(tenant_->id()=1004, group_id_=19, token=0) [2024-06-25 17:00:22.146415] INFO [SERVER.OMT] check_worker_count (ob_tenant.cpp:503) [95435][T1004_L0_G19][T1004][Y0-0000000000000000-0-0] [lt=7] worker thread exit(tenant_->id()=1004, workers_.get_size()=1) [2024-06-25 17:00:22.146424] INFO [SHARE] end_run (ob_tenant_base.cpp:332) [95435][T1004_L0_G19][T0][Y0-0000000000000000-0-0] [lt=6] tenant thread end_run(id_=1004, ret=0, thread_count_=1299) [2024-06-25 17:00:22.146431] INFO unregister_pm (ob_page_manager.cpp:50) [95435][T1004_L0_G19][T0][Y0-0000000000000000-0-0] [lt=5] unregister pm finish(&pm=0x2b3b7a552d80, pm.get_tid()=95435) [2024-06-25 17:00:22.150065] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF7964-0-0] [lt=9] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86d1d470, extra_info="") [2024-06-25 17:00:22.150156] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=80, this=0x2b3a86d273b0, extra_info="palf_id:1007") [2024-06-25 17:00:22.150178] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a86d275e0, extra_info="palf_id:1007") [2024-06-25 17:00:22.150362] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F06C-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1342) [2024-06-25 17:00:22.150372] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F06C-0-0] [lt=6] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a86d1d6a0, extra_info="palf_id:1007") [2024-06-25 17:00:22.150377] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F06C-0-0] [lt=4] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=282, this=0x2b3a86d1d8d0, extra_info="palf_id:1007") [2024-06-25 17:00:22.152491] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABF4538-000616694DDF523E-0-0] [lt=9] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1006}}) [2024-06-25 17:00:22.152507] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABF4538-000616694DDF523E-0-0] [lt=9] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306021871866002, v:0}, arg={tenant_id:1004, ls_id:{id:1006}}, result={tenant_id:1004, ls_id:{id:1006}, cur_readable_scn:{val:1719306021871866002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:22.157235] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B5D-0-0] [lt=6] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:22.161590] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.161600] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:22.161604] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.161609] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.161614] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.161729] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABDC060-0006164325C3014B-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301668, send_timestamp=1719305999860060) [2024-06-25 17:00:22.162009] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABF4533-00061661728FC952-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301557, send_timestamp=1719305999860452) [2024-06-25 17:00:22.168087] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.168094] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.168097] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.168099] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:22.168102] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:22.168143] INFO [LIB] stat (utility.h:1214) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3a8ce1cc70, extra_info="") [2024-06-25 17:00:22.168303] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-000616432547F06E-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301549, send_timestamp=1719305999866753) [2024-06-25 17:00:22.172468] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=0][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:22.172476] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:22.173093] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:22.173102] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.173107] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=3][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=1999711, timeout_timestamp=1719306024172486) [2024-06-25 17:00:22.173114] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:22.173117] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=3][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:22.173121] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=3][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:22.173126] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=5][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:22.173130] WDIAG [SQL] open (ob_result_set.cpp:157) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=3][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:22.173134] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=4][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:22.173137] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=2][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}) [2024-06-25 17:00:22.173143] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=6][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.173149] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=5][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:22.173157] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=7][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:22.173160] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=3][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:22.173162] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:22.173169] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3C-0-0] [lt=5][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:22.173173] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63182][T1002_TenantInf][T1002][YB420ABF4533-000616616F6E9D3C-0-0] [lt=3][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, aret=-4013, ret=-4013) [2024-06-25 17:00:22.173178] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:22.173182] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:22.173187] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:22.173191] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] query failed(ret=-4013, conn=0x2b3ac92dc610, start=1719306022172733, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:22.173197] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:22.173200] WDIAG [SHARE] load_pure_tenant_info_ (ob_tenant_info_proxy.cpp:413) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] failed to read(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", exec_tenant_id=1001, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:22.173222] WDIAG [SHARE] load_tenant_info (ob_tenant_info_proxy.cpp:368) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4013] failed to load purge tenant info(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, for_update=false) [2024-06-25 17:00:22.174247] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63309][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=8] start do ls ha handler(ls_id_array_=[{id:1007}, {id:1010}, {id:1009}, {id:1006}, {id:1008}]) [2024-06-25 17:00:22.177516] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301560, send_timestamp=1719305999875956) [2024-06-25 17:00:22.178309] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.178319] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:22.178324] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:22.178331] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.178336] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.178886] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62247][pnio1][T1001][YB420ABF4533-00061661718F6550-0-0] [lt=10][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:22.181176] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.181184] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.181189] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:22.181195] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.181199] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.181274] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [100368][T1003_Occam][T1003][Y0-0000000000000000-0-0] [lt=10] ====== tenant freeze timer task ====== [2024-06-25 17:00:22.181287] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [100368][T1003_Occam][T1003][Y0-0000000000000000-0-0] [lt=8][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-06-25 17:00:22.182651] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-000616432311A566-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5392, fly_ts=22301554, send_timestamp=1719305999881097) [2024-06-25 17:00:22.182738] INFO [SHARE] get_next_sess_id (ob_active_session_guard.cpp:360) [95801][][T0][Y0-0000000000000000-0-0] [lt=0] succ to generate background session id(sessid=4449401435062272) [2024-06-25 17:00:22.182751] INFO register_pm (ob_page_manager.cpp:40) [95801][][T0][Y0-0000000000000000-0-0] [lt=9] register pm finish(ret=0, &pm=0x2b5135352d80, pm.get_tid()=95801, tenant_id=500) [2024-06-25 17:00:22.182772] INFO [SHARE] pre_run (ob_tenant_base.cpp:314) [95801][][T1004][Y0-0000000000000000-0-0] [lt=6] tenant thread pre_run(MTL_ID()=1004, ret=0, thread_count_=1300) [2024-06-25 17:00:22.182772] INFO [SERVER.OMT] recv_group_request (ob_tenant.cpp:1292) [62233][pnio1][T0][YB420ABDC060-000616432311A566-0-0] [lt=8] worker thread created(id()=1004, group->group_id_=19) [2024-06-25 17:00:22.182778] INFO [RPC.OBRPC] th_init (ob_rpc_translator.cpp:33) [95801][][T1004][Y0-0000000000000000-0-0] [lt=5] Init thread local success [2024-06-25 17:00:22.182784] INFO unregister_pm (ob_page_manager.cpp:50) [95801][][T1004][Y0-0000000000000000-0-0] [lt=4] unregister pm finish(&pm=0x2b5135352d80, pm.get_tid()=95801) [2024-06-25 17:00:22.182789] INFO register_pm (ob_page_manager.cpp:40) [95801][][T1004][Y0-0000000000000000-0-0] [lt=4] register pm finish(ret=0, &pm=0x2b5135352d80, pm.get_tid()=95801, tenant_id=1004) [2024-06-25 17:00:22.182806] INFO [COMMON] process (ob_rpc_processor_simple.cpp:2405) [95801][T1004_L0_G19][T1004][YB420ABDC060-000616432311A566-0-0] [lt=3] handle ObQueryLSIsValidMember requeset(arg_={tenant_id_:1004, self_addr_:"10.189.192.96:2882", ls_array_:[{id:1039}, {id:1005}]}) [2024-06-25 17:00:22.182815] INFO [CLOG] get_member_gc_stat (ob_log_handler.cpp:1509) [95801][T1004_L0_G19][T1004][YB420ABDC060-000616432311A566-0-0] [lt=7] get_member_gc_stat(is_valid_member=true, stat=1, member_list=5{server:"10.186.69.173:2882", timestamp:1716928619820263, flag:0}{server:"10.186.69.177:2882", timestamp:1716928774338690, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, learner_list={learner_num:0, learner_array:[]}, addr="10.189.192.96:2882", this={role:1, proposal_id:1, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1039}) [2024-06-25 17:00:22.182829] INFO [CLOG] get_member_gc_stat (ob_log_handler.cpp:1509) [95801][T1004_L0_G19][T1004][YB420ABDC060-000616432311A566-0-0] [lt=13] get_member_gc_stat(is_valid_member=true, stat=1, member_list=5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, learner_list={learner_num:0, learner_array:[]}, addr="10.189.192.96:2882", this={role:1, proposal_id:3, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1005}) [2024-06-25 17:00:22.184992] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.185000] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:22.185004] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:22.185011] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.185015] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.186901] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.190815] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.190823] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:22.190827] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.190833] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.190837] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.193336] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=2000, remain_us=29838100, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=2, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.200593] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.200602] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.200606] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.200611] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.200615] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.205771] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC0AC-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1, self="10.191.69.51:2882", committed size=4414) [2024-06-25 17:00:22.208041] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=12] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289105330176, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:22.212932] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.212941] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:22.212946] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.212952] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.212958] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.214338] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=19610640, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=100, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:22.215772] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.215780] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:22.215784] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.215790] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.215795] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.218330] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.218338] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.218343] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.218349] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.220656] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 0.93, rt: 107.00) [2024-06-25 17:00:22.220666] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SYS](tenant_id=500, io_status=sys_group_name: INSPECT_BAD_BLOCK_IO, mode: read, size: 2097152.00, iops: 0.93, rt: 780.00) [2024-06-25 17:00:22.220672] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:22.220741] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=17] [IO STATUS SYS](tenant_id=1004, io_status=sys_group_name: SLOG_IO, mode: write, size: 4915.20, iops: 4.66, rt: 66.60) [2024-06-25 17:00:22.220746] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=1004, ref_cnt=2, io_config={group_num_:0, memory_limit_:724775731200, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:100}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=12763552, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:22.220757] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220764] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220767] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220771] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220774] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220778] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220781] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220784] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220788] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220791] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220794] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220799] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220802] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220804] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220808] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.220812] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:22.223692] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.223699] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:22.223703] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.223709] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.223714] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.227051] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC919-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86de7470, extra_info="") [2024-06-25 17:00:22.227063] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC919-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:22.227135] INFO [LIB] stat (utility.h:1197) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=71, this=0x2b3a86df13b0, extra_info="palf_id:1008") [2024-06-25 17:00:22.227146] INFO [LIB] stat (utility.h:1214) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a8ce1daf0, extra_info="") [2024-06-25 17:00:22.227157] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a86df15e0, extra_info="palf_id:1008") [2024-06-25 17:00:22.227900] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F06F-0-0] [lt=6] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a86de76a0, extra_info="palf_id:1008") [2024-06-25 17:00:22.227904] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F06F-0-0] [lt=4] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=838, this=0x2b3a86de78d0, extra_info="palf_id:1008") [2024-06-25 17:00:22.231365] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:22.231397] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:22.231401] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.231404] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.231406] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.231443] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:22.231634] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABDC060-0006164325962B60-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301550, send_timestamp=1719305999930083) [2024-06-25 17:00:22.231833] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-000616431B58ECCB-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5395, fly_ts=22301560, send_timestamp=1719305999930272) [2024-06-25 17:00:22.231857] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95800][T1004_L0_G1][T1004][YB420ABDC060-000616431B58ECCB-0-0] [lt=7] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1039, child={server:"10.189.192.96:2882", region:"sys_region", register_time_us:1719305999930271, keepalive_ts:-1}) [2024-06-25 17:00:22.231866] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95800][T1004_L0_G1][T1004][YB420ABDC060-000616431B58ECCB-0-0] [lt=9] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1039, self="10.191.69.51:2882", child={server:"10.189.192.96:2882", region:"sys_region", register_time_us:1719305999930271, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:22.233536] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.233539] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF5A3F-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b61c69470, extra_info="") [2024-06-25 17:00:22.233544] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:22.233548] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.233553] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.233558] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.233617] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=67, this=0x2b3b61c733b0, extra_info="palf_id:1006") [2024-06-25 17:00:22.233640] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3b61c735e0, extra_info="palf_id:1006") [2024-06-25 17:00:22.233826] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B61-0-0] [lt=5] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b61c696a0, extra_info="palf_id:1006") [2024-06-25 17:00:22.233831] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B61-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=265, this=0x2b3b61c698d0, extra_info="palf_id:1006") [2024-06-25 17:00:22.234114] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:22.235116] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=11][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4383, dropped:767, tid:62272}]) [2024-06-25 17:00:22.237811] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.237822] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.237827] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e75bce0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022183859, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.237850] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=23][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e75bce0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022183859, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.237863] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.237871] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=6][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.237876] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.237878] WDIAG [SQL] open (ob_result_set.cpp:157) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.237882] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.237886] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:22.237891] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=184, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.237897] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=6] will sleep(sleep_us=100000, remain_us=6711370, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=184, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:22.238858] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.238870] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.238875] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6ab5a9e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022184867, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.238896] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=21][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6ab5a9e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022184867, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.238908] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999998, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.238914] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.238917] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.238922] WDIAG [SQL] open (ob_result_set.cpp:157) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.238924] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.238927] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}) [2024-06-25 17:00:22.238933] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, retry_cnt=212, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.238938] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4] will sleep(sleep_us=100000, remain_us=2406402, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=212, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:22.239777] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.241934] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=5] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647483, offset=60298443}, end_cursor:ObLogCursor{file_id=100, log_id=27647484, offset=60298684}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbb8050, len:2869, log_data_len:241, seq:27647483, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:22.242558] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=7][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:224, local_retry_times:224, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:22.242570] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=11][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.242574] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.242579] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:22.242584] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:22.243010] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8015-0-0] [lt=8] push gts task success(*task={:""}) [2024-06-25 17:00:22.245071] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [99399][T1004_LSArchive][T1004][YB420ABF4533-00061642492E6725-0-0] [lt=9] gc stale ls task succ [2024-06-25 17:00:22.247328] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8015-0-0] [lt=9] rpc statistics(total_trans_msg_count=2, total_batch_msg_count=0) [2024-06-25 17:00:22.247933] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=14] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647484, offset=60298684}, end_cursor:ObLogCursor{file_id=100, log_id=27647485, offset=60298933}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbcc050, len:2628, log_data_len:249, seq:27647484, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:22.249015] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.249025] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.249029] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb2177bc0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022195350, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.249049] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=20][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb2177bc0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022195350, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.249061] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999990, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.249067] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.249070] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.249075] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.249078] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.249080] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:22.249086] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=3, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.249091] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4] will sleep(sleep_us=3000, remain_us=29782344, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=3, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.250064] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF7968-0-0] [lt=3] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1342, avg_group_log_size=122) [2024-06-25 17:00:22.250181] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.250191] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.250195] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6d6d1ce0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022196403, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.250213] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=18][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6d6d1ce0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022196403, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.250224] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.250233] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.250238] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.250240] WDIAG [SQL] open (ob_result_set.cpp:157) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.250244] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.250247] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:22.250252] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=80, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.250259] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=6] will sleep(sleep_us=80000, remain_us=22476917, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=80, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:22.251366] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.251375] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.251379] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.251384] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.251389] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.251579] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62749][T1_ReplaySrv6][T1][Y0-0000000000000000-0-0] [lt=8] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:22.252151] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=6][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:3, local_retry_times:3, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:22.252158] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=6][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.252161] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.252163] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:22.252169] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:22.253423] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=9] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647485, offset=60298933}, end_cursor:ObLogCursor{file_id=100, log_id=27647486, offset=60299182}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbb0050, len:2379, log_data_len:249, seq:27647485, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:22.253535] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647486, offset=60299182}, end_cursor:ObLogCursor{file_id=100, log_id=27647487, offset=60299431}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbd4050, len:2130, log_data_len:249, seq:27647486, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:22.253639] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=10] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647487, offset=60299431}, end_cursor:ObLogCursor{file_id=100, log_id=27647488, offset=60299680}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbc4050, len:1881, log_data_len:249, seq:27647487, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:22.254861] INFO eloop_run (eloop.c:143) [62250][pnio2][T0][Y0-0000000000000000-0-0] [lt=8] PNIO [ratelimit] time: 1719306022254861, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007617, add_bytes: 0 [2024-06-25 17:00:22.261215] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:305) [100467][T1003_Flush][T1003][Y0-0000000000000000-0-0] [lt=5] ====== traversal_flush timer task ====== [2024-06-25 17:00:22.261225] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:338) [100467][T1003_Flush][T1003][Y0-0000000000000000-0-0] [lt=7] no logstream(ret=0, ls_cnt=0) [2024-06-25 17:00:22.261913] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABF4538-00061669548FCC6F-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301546, send_timestamp=1719305999960366) [2024-06-25 17:00:22.266787] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [62710][T1_LCLSender][T1][Y0-0000000000000000-0-0] [lt=14] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=30358025, total_destructed_detector=30358025, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x2b399e3de2b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-06-25 17:00:22.268141] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.268148] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.268150] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.268152] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:22.268155] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:22.268371] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-000616432547F071-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301546, send_timestamp=1719305999966824) [2024-06-25 17:00:22.278940] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62241][pnio1][T1001][YB420ABF4533-00061661718F6551-0-0] [lt=7][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.285256] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.285268] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.285752] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.285761] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.285764] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a69342d80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022231134, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.285781] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a69342d80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022231134, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.285792] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.285798] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.285802] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.285804] WDIAG [SQL] open (ob_result_set.cpp:157) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.285808] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.285811] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:22.285816] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=153, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.285821] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=11449002, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=153, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:22.286955] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.289634] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.291491] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.291503] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.291512] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:22.291517] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:22.291522] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:22.291526] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:22.291529] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=2][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:22.291532] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:22.291537] WDIAG [SQL.ENG] inner_get_next_batch_for_tsc (ob_table_scan_op.cpp:2015) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:22.291540] WDIAG [SQL.ENG] inner_get_next_batch (ob_table_scan_op.cpp:2000) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:22.291544] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] get next batch failed(ret=-4383, eval_ctx={batch_idx:0, batch_size:16, max_batch_size:16, frames_:0x2babb4414380}, id=2, op_name="PHY_TABLE_SCAN") [2024-06-25 17:00:22.291552] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1535) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4383] get next batch failed(ret=-4383) [2024-06-25 17:00:22.291555] WDIAG [SQL.ENG] get_next_row_vectorizely (ob_operator.cpp:1463) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] Failed to get_next_row vectorizedly(ret=-4383) [2024-06-25 17:00:22.291560] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_transmit_op.cpp:27) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] get next row from child failed(ret=-4383) [2024-06-25 17:00:22.291563] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] inner get next row failed(ret=-4383, type=14, op="PHY_DIRECT_TRANSMIT", op_id=1) [2024-06-25 17:00:22.291569] WDIAG [SQL.EXE] sync_send_result (ob_remote_executor_processor.cpp:283) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:22.291572] WDIAG [SQL.EXE] execute_remote_plan (ob_remote_executor_processor.cpp:469) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=2][errcode=-4383] sync send result failed(ret=-4383) [2024-06-25 17:00:22.291579] WDIAG [SQL.EXE] process (ob_remote_executor_processor.cpp:969) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] execute remote plan failed(ret=-4383, local_tenant_schema_version=1718209168713032, exec_ctx_.get_task_exec_ctx().get_query_tenant_begin_schema_version()=1718209168713032) [2024-06-25 17:00:22.296250] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.296262] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.296266] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e6a7a80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022241647, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.296286] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=20][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e6a7a80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022241647, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.296298] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.296305] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.296309] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.296312] WDIAG [SQL] open (ob_result_set.cpp:157) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.296317] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=5][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.296320] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:22.296326] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=6][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=225, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.296332] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=408649, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=225, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:22.305712] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.305723] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.305727] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb2174080, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022252185, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.305747] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=20][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb2174080, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022252185, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.305760] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=13][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999990, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.305767] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.305771] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.305774] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.305779] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.305782] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:22.305787] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=4, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.305793] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5] will sleep(sleep_us=4000, remain_us=29725642, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=4, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.307275] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [63477][T1001_ArcSender][T1001][YB420ABF4533-000616423FCE6725-0-0] [lt=8][errcode=-4018] try free send task failed(ret=-4018) [2024-06-25 17:00:22.309829] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:4, local_retry_times:4, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:22.309842] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=12][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.309846] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.309848] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:22.309855] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:22.314404] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=10][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:100, local_retry_times:100, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:22.314415] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.314419] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.314423] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:22.314429] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:22.316105] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.316117] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.320323] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [62671][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=10] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-06-25 17:00:22.323360] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABF4533-0006166171EF4886-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301558, send_timestamp=1719306000021801) [2024-06-25 17:00:22.326891] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB5E-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b74f470, extra_info="") [2024-06-25 17:00:22.326996] INFO [LIB] stat (utility.h:1197) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=75, this=0x2b3b5b7593b0, extra_info="palf_id:1007") [2024-06-25 17:00:22.327008] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=12] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:22.327020] INFO [LIB] stat (utility.h:1214) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3afcbffaf0, extra_info="") [2024-06-25 17:00:22.327038] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x2b3b5b7595e0, extra_info="palf_id:1007") [2024-06-25 17:00:22.327722] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B63-0-0] [lt=6] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b5b74f6a0, extra_info="palf_id:1007") [2024-06-25 17:00:22.327727] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B63-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=831, this=0x2b3b5b74f8d0, extra_info="palf_id:1007") [2024-06-25 17:00:22.330326] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:80, local_retry_times:80, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:22.330337] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.330340] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.330344] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:22.330349] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:22.331426] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:22.331460] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:22.331464] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.331467] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.331470] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.331547] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1005, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:22.333809] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B66-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=81, avg_log_submit_to_slide_time=264) [2024-06-25 17:00:22.333818] INFO [PALF] read (log_cache.cpp:100) [99515][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=12] [PALF STAT HOT CACHE HIT RATE](palf_id=1006, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:22.334170] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:22.337960] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:184, local_retry_times:184, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:22.337972] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.337975] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.337978] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:22.337984] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:22.338803] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=7) [2024-06-25 17:00:22.339000] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=6][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:212, local_retry_times:212, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:22.339013] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=12][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.339017] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.339021] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:22.339027] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:22.339833] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.343358] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABF4533-0006166172BEBECC-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301557, send_timestamp=1719306000041800) [2024-06-25 17:00:22.343790] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63467][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7326, mem_used=12478464) [2024-06-25 17:00:22.351089] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.351100] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.351735] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [100423][T1003_FreInfoRe][T1003][Y0-0000000000000000-0-0] [lt=7] get all snapshots(ret=0, snapshots.count()=0, snapshots=[]) [2024-06-25 17:00:22.352518] INFO [COMMON] generate_mod_stat_task (memory_dump.cpp:218) [62191][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=9] task info(*task={type_:2, dump_all_:false, p_context_:null, slot_idx_:0, dump_tenant_ctx_:false, tenant_id_:0, ctx_id_:0, p_chunk_:null}) [2024-06-25 17:00:22.352530] INFO [COMMON] handle (memory_dump.cpp:498) [62191][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=11] handle dump task(task={type_:2, dump_all_:false, p_context_:null, slot_idx_:0, dump_tenant_ctx_:false, tenant_id_:0, ctx_id_:0, p_chunk_:null}) [2024-06-25 17:00:22.352711] INFO [COMMON] update_check_range (ob_sql_mem_leak_checker.cpp:62) [62191][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=4] update_check_range(min_check_version=595456, max_check_version=595456, global_version=595458) [2024-06-25 17:00:22.359941] INFO [STORAGE.BLKMGR] inspect_bad_block (ob_block_manager.cpp:1712) [62325][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=13] check macro block(block_info={ref_cnt:1, is_free:false, access_time:1713931160307224}, time_interval=5374862052714) [2024-06-25 17:00:22.361646] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325C30153-0-0] [lt=5][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301551, send_timestamp=1719306000060094) [2024-06-25 17:00:22.362593] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62457][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.362957] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62456][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.363074] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62459][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.363458] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62458][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.363502] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.363514] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.363519] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb21ab940, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022309891, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.363541] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=21][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb21ab940, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022309891, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.363553] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999990, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.363560] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.363565] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.363568] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.363572] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.363575] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:22.363581] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=5, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.363586] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4] will sleep(sleep_us=5000, remain_us=29667849, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=5, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.363624] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62460][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.363809] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62455][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.363843] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1BA9-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf09d470, extra_info="") [2024-06-25 17:00:22.363876] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62454][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.363938] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=64, this=0x2b3aaf0a73b0, extra_info="palf_id:1009") [2024-06-25 17:00:22.363961] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=6, this=0x2b3aaf0a75e0, extra_info="palf_id:1009") [2024-06-25 17:00:22.364722] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F074-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf09d6a0, extra_info="palf_id:1009") [2024-06-25 17:00:22.364727] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F074-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=852, this=0x2b3aaf09d8d0, extra_info="palf_id:1009") [2024-06-25 17:00:22.364883] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [62725][T1_LSArchiveMgr][T1][YB420ABF4533-000616423A1E6725-0-0] [lt=7] gc stale ls task succ [2024-06-25 17:00:22.367067] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62467][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367216] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62436][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367273] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62469][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367286] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62473][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367292] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62464][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367296] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62463][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=4] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367303] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62471][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367438] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62470][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367442] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62466][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367582] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62461][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367599] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62439][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367622] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62437][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367680] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62441][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367718] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62438][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367735] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62462][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367739] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62444][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367745] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62434][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367777] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62443][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367844] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62447][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367871] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62452][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367871] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62450][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367872] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62435][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367938] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62468][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.367996] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62445][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.368167] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62465][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.368197] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.368203] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.368206] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.368208] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:22.368210] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:22.368226] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.368237] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.368244] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62451][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.368241] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a678f8d80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022314107, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.368260] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62442][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.368261] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62446][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.368260] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=19][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a678f8d80, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022314107, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.368267] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62448][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.368273] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.368279] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.368284] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.368287] WDIAG [SQL] open (ob_result_set.cpp:157) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.368290] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.368294] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:22.368298] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=101, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.368305] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=6] will sleep(sleep_us=100000, remain_us=19456673, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=101, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:22.368312] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62440][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.368363] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62449][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.368432] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-000616432547F075-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301570, send_timestamp=1719306000066861) [2024-06-25 17:00:22.368647] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=7][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:5, local_retry_times:5, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:22.368655] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=7][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.368658] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.368660] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:22.368666] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:22.369089] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [99384][T1004_LCLSender][T1004][Y0-0000000000000000-0-0] [lt=9] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=30358025, total_destructed_detector=30358025, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x2b3ae6ff42b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-06-25 17:00:22.373388] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62475][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:22.377900] WDIAG [ARCHIVE] do_thread_task_ (ob_archive_sender.cpp:256) [99402][T1004_ArcSender][T1004][YB420ABF4533-00061642495E6725-0-0] [lt=7][errcode=-4018] try free send task failed(ret=-4018) [2024-06-25 17:00:22.378532] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100190][T1004_L0_G0][T1004][YB420ABA45AB-000616C7C5AFD7D9-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:22.380679] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [62735][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[{id:1}]) [2024-06-25 17:00:22.383264] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99946][T1004_L0_G0][T1004][YB420ABA45AB-000616C7C5AFD7D9-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:22.383558] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABF4533-0006166172BEBED1-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301550, send_timestamp=1719306000082008) [2024-06-25 17:00:22.384076] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.384086] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.384091] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6d6c1240, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022329895, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.384109] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6d6c1240, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022329895, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.384120] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.384127] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.384130] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.384133] WDIAG [SQL] open (ob_result_set.cpp:157) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.384137] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.384140] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:22.384145] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=81, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.384151] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5] will sleep(sleep_us=81000, remain_us=22343025, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=81, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:22.385883] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:153, local_retry_times:153, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:22.385892] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=8][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.387015] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.391522] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.391533] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.391537] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e74cfe0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022337219, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.391558] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=21][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e74cfe0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022337219, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.391570] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.391576] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.391581] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.391584] WDIAG [SQL] open (ob_result_set.cpp:157) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.391587] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.391591] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:22.391596] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=185, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.391603] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=6557664, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=185, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:22.392638] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2252703, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=213, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:22.397187] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:116) [99397][T1004_ObTimer][T1004][Y0-0000000000000000-0-0] [lt=6] ====== [tabletchange] timer task ======(GC_CHECK_INTERVAL=5000000) [2024-06-25 17:00:22.397196] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [99397][T1004_ObTimer][T1004][Y0-0000000000000000-0-0] [lt=6] [tabletchange] task check ls(ls->get_ls_id()={id:1039}, tablet_persist_trigger=0) [2024-06-25 17:00:22.397204] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [99397][T1004_ObTimer][T1004][Y0-0000000000000000-0-0] [lt=7] [tabletchange] task check ls(ls->get_ls_id()={id:1005}, tablet_persist_trigger=0) [2024-06-25 17:00:22.397207] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [99397][T1004_ObTimer][T1004][Y0-0000000000000000-0-0] [lt=2] [tabletchange] task check ls(ls->get_ls_id()={id:1006}, tablet_persist_trigger=0) [2024-06-25 17:00:22.397209] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [99397][T1004_ObTimer][T1004][Y0-0000000000000000-0-0] [lt=2] [tabletchange] task check ls(ls->get_ls_id()={id:1007}, tablet_persist_trigger=0) [2024-06-25 17:00:22.397211] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:150) [99397][T1004_ObTimer][T1004][Y0-0000000000000000-0-0] [lt=2] [tabletchange] task check ls(ls->get_ls_id()={id:1008}, tablet_persist_trigger=0) [2024-06-25 17:00:22.397215] INFO [STORAGE] runTimerTask (ob_tablet_gc_service.cpp:228) [99397][T1004_ObTimer][T1004][Y0-0000000000000000-0-0] [lt=3] [tabletchange] succeed to gc_tablet(ret=0, ret="OB_SUCCESS", ls_cnt=5, times=969) [2024-06-25 17:00:22.397856] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.400727] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.404532] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.406719] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A0A2-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1, self="10.191.69.51:2882", total_slide_log_cnt=12, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=93, avg_log_submit_to_slide_time=832) [2024-06-25 17:00:22.406727] INFO [PALF] read (log_cache.cpp:100) [62690][T1_ReplaySrv0][T1][Y0-0000000000000000-0-0] [lt=3] [PALF STAT HOT CACHE HIT RATE](palf_id=1, read_size=4537, hit_cnt=12, read_cnt=12, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:22.408277] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=14] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289107427328, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:22.410362] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.420164] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.422262] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=6000, remain_us=29609173, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=6, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.425875] INFO handle (memory_dump.cpp:626) [62191][MemoryDump][T0][Y0-0000000000000000-0-0] [lt=6] statistics: tenant_cnt: 7, max_chunk_cnt: 524288 tenant_id ctx_id chunk_cnt label_cnt segv_cnt 1 0 828 214 0 1 2 136 1 0 1 4 1 1 0 1 5 149 6 0 1 7 2 2 0 1 8 70 1 0 1 12 10 3 0 1 13 10 1 0 1 14 1 1 0 1 15 1 1 0 1 18 26 3 0 1 20 1 1 0 500 0 180 248 0 500 7 10 3 0 500 8 79 1 0 500 9 4 1 0 500 10 7 2 0 500 16 5 2 0 500 17 65 7 0 500 22 73 57 0 500 23 105 11 0 508 0 6 13 0 508 8 13 1 0 1001 0 76 147 0 1001 5 23 6 0 1001 7 2 2 0 1001 8 42 1 0 1001 12 1 1 0 1001 14 1 1 0 1002 0 195 215 0 1002 2 2 1 0 1002 3 3 1 0 1002 4 3 1 0 1002 5 38 5 0 1002 8 49 1 0 1002 12 17 3 0 1002 13 5 1 0 1002 14 1 1 0 1002 16 6 4 0 1002 18 34 3 0 1002 20 2 1 0 1003 0 189 143 0 1003 5 110 6 0 1003 7 2 2 0 1003 8 63 1 0 1003 12 1 1 0 1003 14 1 1 0 1004 0 9705 228 0 1004 2 26453 1 0 1004 4 3 1 0 1004 5 566 6 0 1004 7 2 2 0 1004 8 326 1 0 1004 12 79 3 0 1004 13 154 1 0 1004 14 1 2 0 1004 15 3 1 0 1004 18 971 3 0 1004 20 10 1 0 cost_time: 73255 [2024-06-25 17:00:22.427836] INFO [LIB] stat (utility.h:1197) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=11] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=7, this=0x2b3ae6e6c8f8, extra_info="") [2024-06-25 17:00:22.427846] INFO [LIB] stat (utility.h:1197) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=1, this=0x2b3ae6e6cb28, extra_info="") [2024-06-25 17:00:22.429719] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-0006164325210E43-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=330, fly_ts=22301555, send_timestamp=1719306000128164) [2024-06-25 17:00:22.431472] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:22.431507] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:22.431512] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.431514] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.431517] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.431708] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164325962B69-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301568, send_timestamp=1719306000130139) [2024-06-25 17:00:22.432225] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=24] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:22.434188] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:22.438007] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.439740] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=11295084, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=154, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:22.439903] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=14][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.445941] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.445959] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e696fe0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221681243, assoc_session_id:3221681243, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306022177555, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3a97f75ff8, id:3221681243, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e696fe0}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:22.450015] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=254966, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=226, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:22.451736] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62744][T1_ReplaySrv1][T1][Y0-0000000000000000-0-0] [lt=10] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:22.455580] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABA45B0-00061644F03C36DD-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1008}}) [2024-06-25 17:00:22.455596] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABA45B0-00061644F03C36DD-0-0] [lt=9] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306022141368002, v:0}, arg={tenant_id:1004, ls_id:{id:1008}}, result={tenant_id:1004, ls_id:{id:1008}, cur_readable_scn:{val:1719306022141368002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:22.456523] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABF00-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b6d5470, extra_info="") [2024-06-25 17:00:22.456616] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=72, this=0x2b3b5b6df3b0, extra_info="palf_id:1008") [2024-06-25 17:00:22.456641] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3b5b6df5e0, extra_info="palf_id:1008") [2024-06-25 17:00:22.457423] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B6C-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b5b6d56a0, extra_info="palf_id:1008") [2024-06-25 17:00:22.457427] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B6C-0-0] [lt=4] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=839, this=0x2b3b5b6d58d0, extra_info="palf_id:1008") [2024-06-25 17:00:22.461733] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164325C30157-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301558, send_timestamp=1719306000160174) [2024-06-25 17:00:22.462025] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABF4533-00061661728FC95D-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301554, send_timestamp=1719306000160470) [2024-06-25 17:00:22.462481] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [63471][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=4] ====== check clog disk timer task ====== [2024-06-25 17:00:22.462490] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [63471][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=8] get_disk_usage(ret=0, capacity(MB):=614, used(MB):=0) [2024-06-25 17:00:22.462497] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [63471][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=4] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=193273528, need_update_checkpoint_scn=false) [2024-06-25 17:00:22.463727] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100576][T1003_L0_G0][T1003][YB420ABA45AB-000616C80A1FC7F8-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1007}}) [2024-06-25 17:00:22.463744] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100576][T1003_L0_G0][T1004][YB420ABA45AB-000616C80A1FC7F8-0-0] [lt=9] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306022047759003, v:0}, arg={tenant_id:1004, ls_id:{id:1007}}, result={tenant_id:1004, ls_id:{id:1007}, cur_readable_scn:{val:1719306022047759003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:22.464261] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=9] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-06-25 17:00:22.464284] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=9] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1002, need_refresh=false, gts_local_cache={srr:[mts=1719306022177830], gts:1719306022178118000, latest_srr:[mts=1719306022464279]}) [2024-06-25 17:00:22.464646] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62236][pnio1][T0][YB420ABF4533-0006166171AF65F7-0-0] [lt=10][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:22.464655] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62236][pnio1][T0][YB420ABF4533-0006166171AF65F7-0-0] [lt=8] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:22.464668] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62236][pnio1][T0][YB420ABF4533-0006166171AF65F7-0-0] [lt=5] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306022464274]}) [2024-06-25 17:00:22.465211] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.468210] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.468218] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.468221] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.468223] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:22.468227] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:22.468367] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.468453] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-000616432547F079-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301551, send_timestamp=1719306000166901) [2024-06-25 17:00:22.471759] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.478962] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62233][pnio1][T1001][YB420ABF4533-00061661718F6553-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.481020] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.481984] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=1] will sleep(sleep_us=7000, remain_us=29549451, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=7, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.486765] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [100471][T1003_LSArchive][T1003][YB420ABF4533-0006164249CE6725-0-0] [lt=7] gc stale ls task succ [2024-06-25 17:00:22.487072] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.491664] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=6][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:185, local_retry_times:185, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:22.491676] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.491679] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.491682] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:22.491689] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:22.492700] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=8][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.493671] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [100481][T1003_HAService][T1003][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:22.494794] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63304][T1002_ArcSrv][T1001][YB420ABF4533-000616423F3E6725-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.499923] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [99409][T1004_HAService][T1004][Y0-0000000000000000-0-0] [lt=8] start do ls ha handler(ls_id_array_=[{id:1039}, {id:1005}, {id:1008}, {id:1007}, {id:1006}]) [2024-06-25 17:00:22.503066] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [100470][T1003_TabletShe][T1003][Y0-0000000000000000-0-0] [lt=5] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-06-25 17:00:22.503074] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:101) [100470][T1003_TabletShe][T1003][Y0-0000000000000000-0-0] [lt=6] [emptytablet] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, times=2364) [2024-06-25 17:00:22.519009] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=82000, remain_us=22208167, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=82, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:22.522055] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=19302923, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=102, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:22.527254] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=3] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:22.531514] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:22.531542] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=17, this=0x2b3b5a817470, extra_info="") [2024-06-25 17:00:22.531551] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5a884e50, extra_info="id:1005") [2024-06-25 17:00:22.531563] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=12] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:22.531566] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.531568] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.531571] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.531633] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=11] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=74, this=0x2b3b5a8213b0, extra_info="palf_id:1005") [2024-06-25 17:00:22.531652] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b5a8215e0, extra_info="palf_id:1005") [2024-06-25 17:00:22.531760] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325962B6E-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301556, send_timestamp=1719306000230204) [2024-06-25 17:00:22.531827] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B6F-0-0] [lt=3] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8176a0, extra_info="palf_id:1005") [2024-06-25 17:00:22.531834] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B6F-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=294, this=0x2b3b5a8178d0, extra_info="palf_id:1005") [2024-06-25 17:00:22.531975] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABDC060-000616431B58ECCD-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5395, fly_ts=22301554, send_timestamp=1719306000230420) [2024-06-25 17:00:22.531994] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95800][T1004_L0_G1][T1004][YB420ABDC060-000616431B58ECCD-0-0] [lt=3] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1005, child={server:"10.189.192.96:2882", region:"sys_region", register_time_us:1719306000230416, keepalive_ts:-1}) [2024-06-25 17:00:22.532001] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95800][T1004_L0_G1][T1004][YB420ABDC060-000616431B58ECCD-0-0] [lt=6] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1005, self="10.191.69.51:2882", child={server:"10.189.192.96:2882", region:"sys_region", register_time_us:1719306000230416, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:22.533558] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95800][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF5A4B-0-0] [lt=5] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:22.534220] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:22.536147] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [100464][T1003_ReqMemEvi][T1003][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1003, evict_high_mem_level=1932735283, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=83025, mem_used=195495936) [2024-06-25 17:00:22.537602] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.538870] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=10] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=6) [2024-06-25 17:00:22.539864] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABDC061-00061643952B83BF-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=50620703, send_timestamp=1719305971919160) [2024-06-25 17:00:22.539941] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=5][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.542754] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=8000, remain_us=29488682, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=8, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.545258] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.545269] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:22.545273] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e7581a0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022490777, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.545291] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=18][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e7581a0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306022490777, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:22.545303] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:22.545309] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:22.545312] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:22.545316] WDIAG [SQL] open (ob_result_set.cpp:157) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:22.545320] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:22.545323] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:22.545328] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=186, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:22.545334] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=6403933, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=186, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:22.546409] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=2098931, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=214, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:22.547597] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [100482][T1003_TxLoopWor][T1003][Y0-0000000000000000-0-0] [lt=3] tx gc loop thread is running(MTL_ID()=1003) [2024-06-25 17:00:22.547603] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [100482][T1003_TxLoopWor][T1003][Y0-0000000000000000-0-0] [lt=6] try gc retain ctx [2024-06-25 17:00:22.550079] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=9][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.550371] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F07B-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=96, avg_log_submit_to_slide_time=285) [2024-06-25 17:00:22.550376] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:22.550889] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301560, send_timestamp=1719306000249329) [2024-06-25 17:00:22.551654] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.554492] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.556652] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:22.557907] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48FE-0-0] [lt=5][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=2513, pos=316, bitmap=NULL) [2024-06-25 17:00:22.558089] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48FE-0-0] [lt=10][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=2, len=71, pos=50, bitmap=NULL) [2024-06-25 17:00:22.558295] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F48FE-0-0] [lt=6][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=57, len=308, pos=278, bitmap=NULL) [2024-06-25 17:00:22.558408] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.561772] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325C3015B-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301553, send_timestamp=1719306000260218) [2024-06-25 17:00:22.562074] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABF4533-00061661728FC961-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301559, send_timestamp=1719306000260515) [2024-06-25 17:00:22.562116] INFO [LIB] stat (utility.h:1197) [63319][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=4] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a872048f8, extra_info="") [2024-06-25 17:00:22.562125] INFO [LIB] stat (utility.h:1197) [63319][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=1, this=0x2b3a87204b28, extra_info="") [2024-06-25 17:00:22.564236] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.568235] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.568242] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.568245] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.568247] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:22.568250] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:22.574073] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.574969] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [63485][T1001_TxLoopWor][T1001][Y0-0000000000000000-0-0] [lt=3] tx gc loop thread is running(MTL_ID()=1001) [2024-06-25 17:00:22.574976] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [63485][T1001_TxLoopWor][T1001][Y0-0000000000000000-0-0] [lt=7] try gc retain ctx [2024-06-25 17:00:22.576785] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [63254][T1002_ReqMemEvi][T1002][YB420ABF4533-0006165F60BFEA4C-0-0] [lt=0][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.576798] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63254][T1002_ReqMemEvi][T1002][YB420ABF4533-0006165F60BFEA4C-0-0] [lt=0] will sleep(sleep_us=9000, remain_us=9072460, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=9, v.err_=-4383, timeout_timestamp=1719306031649258) [2024-06-25 17:00:22.579083] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62239][pnio1][T1001][YB420ABF4533-00061661718F6554-0-0] [lt=7][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.582540] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=24] get_max_applied_scn(scn={val:1719306021963576004, v:0}, this={ls_id_:{id:1039}, role_:1, proposal_id_:1, palf_committed_end_lsn_:{lsn:2854440585}, last_check_scn_:{val:1719306022464265000, v:0}, max_applied_cb_scn_:{val:1719306021963576004, v:0}}) [2024-06-25 17:00:22.582554] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1018) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=13] get_min_unreplayed_log_info in skip state(lsn={lsn:0}, scn={val:1716928602493732000, v:0}, this={ls_id_:{id:1039}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1716928608957087, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:0}, next_to_submit_scn_:{val:0, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1716928602493732000, v:0}, iterator_:{iterator_impl:{buf_:null, next_round_pread_size:3678208, curr_read_pos:0, curr_read_buf_start_pos:0, curr_read_buf_end_pos:0, log_storage_:{IteratorStorage:{start_lsn:{lsn:0}, end_lsn:{lsn:0}, read_buf:{buf_len_:0, buf_:null}, block_size:67104768, log_storage_:0x2bcde052b470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:18446744073709551615, v:3}, curr_entry:{LogEntryHeader:{magic:0, version:0, log_size:-1, scn_:{val:18446744073709551615, v:3}, data_checksum:0, flag:0}}, init_mode_version:0, accumulate_checksum:-1, curr_entry_is_padding:0, padding_entry_size:0, padding_entry_scn:{val:18446744073709551615, v:3}}}}}) [2024-06-25 17:00:22.582590] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=31] get_max_applied_scn(scn={val:1719306021963576004, v:0}, this={ls_id_:{id:1005}, role_:1, proposal_id_:3, palf_committed_end_lsn_:{lsn:5678775314487}, last_check_scn_:{val:1719306022464265000, v:0}, max_applied_cb_scn_:{val:1719306021963576004, v:0}}) [2024-06-25 17:00:22.582598] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=6] get_min_unreplayed_log_info(lsn={lsn:1750156420456}, scn={val:1716561397833071015, v:0}, this={ls_id_:{id:1005}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1716561398006517, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:11603, ret_code_:0}}, next_to_submit_lsn_:{lsn:1750156420456}, next_to_submit_scn_:{val:1716561397833071015, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713332543407963001, v:0}, iterator_:{iterator_impl:{buf_:0x2b66c950f000, next_round_pread_size:3678208, curr_read_pos:50242, curr_read_buf_start_pos:0, curr_read_buf_end_pos:50242, log_storage_:{IteratorStorage:{start_lsn:{lsn:1750156370214}, end_lsn:{lsn:1750156420456}, read_buf:{buf_len_:294912, buf_:0x2b66c950f000}, block_size:67104768, log_storage_:0x2b3b5a81d470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1716561397833071014, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:32195, scn_:{val:1716561397833071014, v:0}, data_checksum:3072636752, flag:1}}, init_mode_version:0, accumulate_checksum:2076199808, curr_entry_is_padding:0, padding_entry_size:32227, padding_entry_scn:{val:1716561397833071014, v:0}}}}}) [2024-06-25 17:00:22.582617] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=17] get_max_applied_scn(scn={val:1713332543409019001, v:0}, this={ls_id_:{id:1006}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713332543409019001, v:0}}) [2024-06-25 17:00:22.582625] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=6] get_min_unreplayed_log_info(lsn={lsn:9248068442662}, scn={val:1719306022372701002, v:0}, this={ls_id_:{id:1006}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306022533820, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:269502, ret_code_:0}}, next_to_submit_lsn_:{lsn:9248068442662}, next_to_submit_scn_:{val:1719306022372701002, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713332543409019001, v:0}, iterator_:{iterator_impl:{buf_:0x2bcce2c05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:9248068442540}, end_lsn:{lsn:9248068442662}, read_buf:{buf_len_:3682304, buf_:0x2bcce2c05000}, block_size:67104768, log_storage_:0x2b3b61c6f470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306022372701001, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306022372701001, v:0}, data_checksum:682552663, flag:0}}, init_mode_version:0, accumulate_checksum:3190753299, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306022372701001, v:0}}}}}) [2024-06-25 17:00:22.587126] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.591784] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.592804] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1044) [62671][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=11] success to update min reserved snapshot(reserved_snapshot=1719304221818827000, duration=1800, snapshot_gc_ts_=1719306021818827000) [2024-06-25 17:00:22.592819] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1071) [62671][T1_FreInfoReloa][T1][Y0-0000000000000000-0-0] [lt=11] update reserved snapshot finished(cost_ts=15, reserved_snapshot=1719304221818827000) [2024-06-25 17:00:22.593629] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=11141195, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=155, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:22.593899] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.598926] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.603764] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=101217, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=227, v.err_=-4383, timeout_timestamp=1719306022704981) [2024-06-25 17:00:22.604342] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=9000, remain_us=29427093, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=9, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.604833] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.608507] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289107427328, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:22.610840] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [62703][T1_LSMetaCh][T1][YB420ABF4533-0006166172AE6CE5-0-0] [lt=45] fetch root ls success(local_cluster_id=1713323324, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.171:2882", sql_port:2881, role:1, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607828518, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.176:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:4, zone:"zone4", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607829360, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.189.192.96:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:5, zone:"zone5", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607830236, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.51:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607831113, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.56:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["10.186.69.171:2882", "10.191.69.51:2882", "10.191.69.56:2882", "10.186.69.176:2882", "10.189.192.96:2882"]) [2024-06-25 17:00:22.614638] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.617226] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=4][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.617240] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [27554][T1002_L0_G0][T1002][YB420ABF4533-00061661700E6767-0-0] [lt=1] will sleep(sleep_us=34000, remain_us=521693490, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=34, v.err_=-4383, timeout_timestamp=1719306544310730) [2024-06-25 17:00:22.618072] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63299][T1002_LSArchive][T1002][YB420ABF4533-000616423EEE6725-0-0] [lt=9] gc stale ls task succ [2024-06-25 17:00:22.618434] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63484][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=7] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:22.625590] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.631567] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:22.631597] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:22.631603] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.631605] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.631608] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.631819] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC060-0006164325962B74-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301553, send_timestamp=1719306000330266) [2024-06-25 17:00:22.632460] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.633575] INFO [STORAGE.TRANS] alloc (ob_trans_factory.cpp:141) [62747][T1_ReplaySrv4][T1][Y0-0000000000000000-0-0] [lt=13] ObTransCtx statistics(active_coord_ctx_count=0, active_part_ctx_count=1, total_release_part_ctx_count=39) [2024-06-25 17:00:22.633617] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62747][T1_ReplaySrv4][T1][Y0-0000000000000000-0-0] [lt=10] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:22.634275] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:22.637408] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [62676][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=7] run rewrite rule refresh task(rule_mgr_->tenant_id_=1) [2024-06-25 17:00:22.639996] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:120) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7] tenant info is invalid(ret=-4076, query_ts=0, standby_scn={val:0, v:0}, this={inited:true, last_id:-1, tenant_id:1004, epoch:-1, self:"10.191.69.51:2882", switch_to_leader_ts:-1}) [2024-06-25 17:00:22.640008] INFO [STORAGE.TRANS] query_and_update_last_id (ob_standby_timestamp_service.cpp:134) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=11] ObStandbyTimestampService state(*this={inited:true, last_id:-1, tenant_id:1004, epoch:-1, self:"10.191.69.51:2882", switch_to_leader_ts:-1}) [2024-06-25 17:00:22.640012] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=4][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.645394] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:186, local_retry_times:186, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:22.645403] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.645407] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:22.645409] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:22.645415] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:22.651728] INFO [LIB] stat (utility.h:1197) [62749][T1_ReplaySrv6][T1][Y0-0000000000000000-0-0] [lt=7] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=16, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a3f4048f8, extra_info="") [2024-06-25 17:00:22.651738] INFO [LIB] stat (utility.h:1197) [62749][T1_ReplaySrv6][T1][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=16, stat_interval=1000000, avg (count/cost)=43, this=0x2b3a3f404b28, extra_info="") [2024-06-25 17:00:22.654854] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=1] will sleep(sleep_us=83000, remain_us=22072321, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=83, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:22.656632] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95800][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABF08-0-0] [lt=5] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:22.659530] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=3] ====== checkpoint timer task ====== [2024-06-25 17:00:22.660006] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=9] succ to get rec scn(*this={this:0x2bcefe0b6030, ls_id:{id:1039}, tenant_id:1004, state:"L_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1716928615592893007, v:0}, last_push_gc_task_ts_:1716930408791203, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:93}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:22.660022] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=15] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3b5a9b38c0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1716930408792227, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3b5a9b38c0, snapshot_version:{val:1716930408792227, v:0}, ls_id:{id:1039}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:22.660040] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=13] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1716928615592893007, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1716928615592893007, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1039}) [2024-06-25 17:00:22.660062] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=9] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b589fc92510, is_inited:true, freezing_scn:{val:0, v:0}, ls:{ls_meta:{tenant_id:1004, ls_id:{id:1039}, ls_create_status:1, clog_checkpoint_scn:{val:1719306017459730000, v:0}, clog_base_lsn:{lsn:2818400256}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306019870754003, v:0}, tablet_change_checkpoint_scn:{val:1719301157002134002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:1, proposal_id:1, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1039}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1039, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:50335808, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:22.660097] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=33] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1039}) [2024-06-25 17:00:22.660114] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=7] get_binary_search_range_(ret=0, this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=30, max_block_id=42, result_block_id=274894685184, committed_lsn={lsn:2854440707}, scn={val:1719306022464265000, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:22.660630] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647488, offset=60299680}, end_cursor:ObLogCursor{file_id=100, log_id=27647489, offset=60299921}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbbc050, len:1632, log_data_len:241, seq:27647488, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:22.660647] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=9] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:2818400256}, checkpoint_scn={val:1719306022464265000, v:0}, ls_id={id:1039}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:22.660659] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:2818400256}, new_base_lsn={lsn:2818400256}, lsn={lsn:2818400256}) [2024-06-25 17:00:22.660666] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=7] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:2818400256}, ls->get_ls_id()={id:1039}) [2024-06-25 17:00:22.661133] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=4] succ to get rec scn(*this={this:0x2b3b62604030, ls_id:{id:1005}, tenant_id:1004, state:"L_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718607258630470000, v:0}, last_push_gc_task_ts_:1718607262546387, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:1155}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:22.661141] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=8] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3b5a9b0080, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718607262547397, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3b5a9b0080, snapshot_version:{val:1718607262547397, v:0}, ls_id:{id:1005}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:22.661151] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=9] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1718098646195382000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1718098646195382000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1005}) [2024-06-25 17:00:22.661170] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=8] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3b5a880510, is_inited:true, freezing_scn:{val:1718180641483667000, v:0}, ls:{ls_meta:{tenant_id:1004, ls_id:{id:1005}, ls_create_status:1, clog_checkpoint_scn:{val:1718725371184341000, v:0}, clog_base_lsn:{lsn:5648141217792}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306019870754003, v:0}, tablet_change_checkpoint_scn:{val:1719301157002134002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716899681409542026, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:1, proposal_id:3, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1005}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1005, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:22.661196] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=24] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x2b3b5a8801b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x2b3b5a885cd0, table_type:1, memtable_head:158, memtable_tail:159, t3m:0x2b3ac3eb2030, tables:[null, null, null, null, null, null, null, null, null, null, null, null, null, null, 0x2b3b5a83adf0, null]}, is_freezing:false, ls_id:{id:1005}, mini_merge_recycle_commit_versions_ts:1718725370737262, tx_data_table:0x2b3b5a8871d0, ls_tablet_svr:0x2b3b5a880190, slice_allocator:0x2b3b5a887210}, min_rec_scn={val:1718725371184341000, v:0}, ls_id_={id:1005}) [2024-06-25 17:00:22.661211] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=13] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1718725371184341000, v:0}, checkpoint_scn_in_ls_meta={val:1718725371184341000, v:0}, ls_id={id:1005}, service_type="TRANS_SERVICE") [2024-06-25 17:00:22.661218] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:5648141217792}, new_base_lsn={lsn:5648141217792}, lsn={lsn:5648141217792}) [2024-06-25 17:00:22.661223] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=4] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:5648141217792}, ls->get_ls_id()={id:1005}) [2024-06-25 17:00:22.661688] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=4] succ to get rec scn(*this={this:0x2b3b626b6030, ls_id:{id:1006}, tenant_id:1004, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718844879025332000, v:0}, last_push_gc_task_ts_:1718846540544282, skip_remove_cnt_:0}, aggre_rec_scn:{val:1718846540868570004, v:0}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:1176}, aggre_rec_scn={val:1718846540868570004, v:0}) [2024-06-25 17:00:22.661696] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=8] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3b5a9b0e90, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718846540545309, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3b5a9b0e90, snapshot_version:{val:1718846540545309, v:0}, ls_id:{id:1006}, is_frozen:false}, rec_scn={val:1718846540868570004, v:0}) [2024-06-25 17:00:22.661704] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=7] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1718098914567681000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1718098914567681000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1006}) [2024-06-25 17:00:22.661722] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=6] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3b61c2c510, is_inited:true, freezing_scn:{val:1718738317058375001, v:0}, ls:{ls_meta:{tenant_id:1004, ls_id:{id:1006}, ls_create_status:1, clog_checkpoint_scn:{val:1718846540868570004, v:0}, clog_base_lsn:{lsn:8904332980224}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306019870754003, v:0}, tablet_change_checkpoint_scn:{val:1719301161935089002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716928614029017002, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1006}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1006, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:22.661743] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=20] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_CTX_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObITable:{this:0x2b3b5a9b0e90, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718846540545309, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3b5a9b0e90, snapshot_version:{val:1718846540545309, v:0}, ls_id:{id:1006}, is_frozen:false}, min_rec_scn={val:1718846540868570004, v:0}, ls_id_={id:1006}) [2024-06-25 17:00:22.661752] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=7] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1718846540868570004, v:0}, checkpoint_scn_in_ls_meta={val:1718846540868570004, v:0}, ls_id={id:1006}, service_type="TRANS_SERVICE") [2024-06-25 17:00:22.661757] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=4][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:8904332980224}, new_base_lsn={lsn:8904332980224}, lsn={lsn:8904332980224}) [2024-06-25 17:00:22.661761] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=4] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:8904332980224}, ls->get_ls_id()={id:1006}) [2024-06-25 17:00:22.661823] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164325C3015F-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301550, send_timestamp=1719306000360272) [2024-06-25 17:00:22.661928] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1010, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:22.662116] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABF4533-00061661728FC965-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301551, send_timestamp=1719306000360565) [2024-06-25 17:00:22.662130] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC965-0-0] [lt=10] [PALF STAT COMMITTED LOG SIZE](palf_id=1010, self="10.191.69.51:2882", committed size=1342) [2024-06-25 17:00:22.662226] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=3] succ to get rec scn(*this={this:0x2b3b62004030, ls_id:{id:1007}, tenant_id:1004, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718625962894074001, v:0}, last_push_gc_task_ts_:1718626273064530, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:1162}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:22.662236] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=10] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3b5a9b1ca0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718626273065554, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3b5a9b1ca0, snapshot_version:{val:1718626273065554, v:0}, ls_id:{id:1007}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:22.662242] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=6] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1718625962894074001, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1718625962894074001, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1007}) [2024-06-25 17:00:22.662259] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=7] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3b5b798510, is_inited:true, freezing_scn:{val:1718625962894074001, v:0}, ls:{ls_meta:{tenant_id:1004, ls_id:{id:1007}, ls_create_status:1, clog_checkpoint_scn:{val:1718726419050442000, v:0}, clog_base_lsn:{lsn:5781478391808}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306019870754003, v:0}, tablet_change_checkpoint_scn:{val:1719301167059506001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716901297164071002, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1007}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1007, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:22.662280] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=20] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x2b3b5b7981b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x2b3b5b79dcd0, table_type:1, memtable_head:154, memtable_tail:155, t3m:0x2b3ac3eb2030, tables:[null, null, null, null, null, null, null, null, null, null, 0x2b3b5a8381b0, null, null, null, null, null]}, is_freezing:false, ls_id:{id:1007}, mini_merge_recycle_commit_versions_ts:1718725371783969, tx_data_table:0x2b3b5b79f1d0, ls_tablet_svr:0x2b3b5b798190, slice_allocator:0x2b3b5b79f210}, min_rec_scn={val:1718726419050442000, v:0}, ls_id_={id:1007}) [2024-06-25 17:00:22.662290] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=9] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1718726419050442000, v:0}, checkpoint_scn_in_ls_meta={val:1718726419050442000, v:0}, ls_id={id:1007}, service_type="TRANS_SERVICE") [2024-06-25 17:00:22.662294] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=3][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:5781478391808}, new_base_lsn={lsn:5781478391808}, lsn={lsn:5781478391808}) [2024-06-25 17:00:22.662300] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=5] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:5781478391808}, ls->get_ls_id()={id:1007}) [2024-06-25 17:00:22.662761] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=2] succ to get rec scn(*this={this:0x2b3b620b6030, ls_id:{id:1008}, tenant_id:1004, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1719194229310671000, v:0}, last_push_gc_task_ts_:1719195996691491, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:1184}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:22.662768] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=7] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3b5a9b2ab0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1719195996692534, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3b5a9b2ab0, snapshot_version:{val:1719195996692534, v:0}, ls_id:{id:1008}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:22.662776] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=7] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:1718240890377102000, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:1718240890377102000, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1008}) [2024-06-25 17:00:22.662792] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=5] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3b5b6a4510, is_inited:true, freezing_scn:{val:1718844273612110000, v:0}, ls:{ls_meta:{tenant_id:1004, ls_id:{id:1008}, ls_create_status:1, clog_checkpoint_scn:{val:1719196007356667000, v:0}, clog_base_lsn:{lsn:5845831864320}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306019870754003, v:0}, tablet_change_checkpoint_scn:{val:1719301171837486004, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716901298196291003, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1008}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1008, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:7936, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:22.662812] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=19] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x2b3b5b6a41b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x2b3b5b6a9cd0, table_type:1, memtable_head:162, memtable_tail:163, t3m:0x2b3ac3eb2030, tables:[null, null, 0x2b3b5a838cc0, null, null, null, null, null, null, null, null, null, null, null, null, null]}, is_freezing:false, ls_id:{id:1008}, mini_merge_recycle_commit_versions_ts:1719194197629200, tx_data_table:0x2b3b5b6ab1d0, ls_tablet_svr:0x2b3b5b6a4190, slice_allocator:0x2b3b5b6ab210}, min_rec_scn={val:1719196007356667000, v:0}, ls_id_={id:1008}) [2024-06-25 17:00:22.662820] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=7] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1719196007356667000, v:0}, checkpoint_scn_in_ls_meta={val:1719196007356667000, v:0}, ls_id={id:1008}, service_type="TRANS_SERVICE") [2024-06-25 17:00:22.662825] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=4][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:5845831864320}, new_base_lsn={lsn:5845831864320}, lsn={lsn:5845831864320}) [2024-06-25 17:00:22.662830] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=5] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:5845831864320}, ls->get_ls_id()={id:1008}) [2024-06-25 17:00:22.662833] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:191) [99394][T1004_TxCkpt][T1004][Y0-0000000000000000-0-0] [lt=3] succeed to update_clog_checkpoint(ret=0, ls_cnt=5) [2024-06-25 17:00:22.666230] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.667100] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=10000, remain_us=29364336, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=10, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.668321] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=20, this=0x2b3a9537b470, extra_info="") [2024-06-25 17:00:22.668335] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=9, avg_log_batch_cnt=1, total_group_log_size=1098, avg_group_log_size=122) [2024-06-25 17:00:22.668343] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT APPEND DATA SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1098) [2024-06-25 17:00:22.668350] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT APPEND COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a9534ee50, extra_info="id:1006") [2024-06-25 17:00:22.668353] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.668358] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.668360] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.668363] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:22.668366] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:22.668404] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=59, this=0x2b3a953853b0, extra_info="palf_id:1006") [2024-06-25 17:00:22.668411] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:22.668425] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=1098) [2024-06-25 17:00:22.668435] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=4, this=0x2b3a953855e0, extra_info="palf_id:1006") [2024-06-25 17:00:22.668587] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-000616432547F080-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301557, send_timestamp=1719306000367029) [2024-06-25 17:00:22.668634] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F080-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a9537b6a0, extra_info="palf_id:1006") [2024-06-25 17:00:22.668642] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F080-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=299, this=0x2b3a9537b8d0, extra_info="palf_id:1006") [2024-06-25 17:00:22.668645] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F080-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=25, avg_log_submit_to_flush_time=74, avg_log_submit_to_slide_time=274) [2024-06-25 17:00:22.675806] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=19149173, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=103, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:22.679081] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62238][pnio1][T1001][YB420ABF4533-00061661718F6555-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.682595] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=15] get_max_applied_scn(scn={val:1713332543410073001, v:0}, this={ls_id_:{id:1007}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713332543410073001, v:0}}) [2024-06-25 17:00:22.682608] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=12] get_min_unreplayed_log_info(lsn={lsn:5896353987180}, scn={val:1719306022548511001, v:0}, this={ls_id_:{id:1007}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306022627915, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:237788, ret_code_:0}}, next_to_submit_lsn_:{lsn:5896353987180}, next_to_submit_scn_:{val:1719306022548511001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713332543410073001, v:0}, iterator_:{iterator_impl:{buf_:0x2b4f5c205000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:5896353987058}, end_lsn:{lsn:5896353987180}, read_buf:{buf_len_:3682304, buf_:0x2b4f5c205000}, block_size:67104768, log_storage_:0x2b3b5b755470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306022548511000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306022548511000, v:0}, data_checksum:2442502576, flag:1}}, init_mode_version:0, accumulate_checksum:4100854471, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306022548511000, v:0}}}}}) [2024-06-25 17:00:22.682635] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=25] get_max_applied_scn(scn={val:1713332543412178000, v:0}, this={ls_id_:{id:1008}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713332543412178000, v:0}}) [2024-06-25 17:00:22.682641] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [99393][T1004_TenantWea][T1004][Y0-0000000000000000-0-0] [lt=5] get_min_unreplayed_log_info(lsn={lsn:5880977361956}, scn={val:1719306022642223001, v:0}, this={ls_id_:{id:1008}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306022657490, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:228213, ret_code_:0}}, next_to_submit_lsn_:{lsn:5880977361956}, next_to_submit_scn_:{val:1719306022642223001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713332543412178000, v:0}, iterator_:{iterator_impl:{buf_:0x2b4f56c05000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:5880977361834}, end_lsn:{lsn:5880977361956}, read_buf:{buf_len_:3682304, buf_:0x2b4f56c05000}, block_size:67104768, log_storage_:0x2b3b5b6db470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306022642223000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306022642223000, v:0}, data_checksum:3647337712, flag:0}}, init_mode_version:0, accumulate_checksum:58354610, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306022642223000, v:0}}}}}) [2024-06-25 17:00:22.687181] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.691243] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.698907] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:22.698919] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=11][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.698929] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=6250338, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=187, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:22.700091] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1945249, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=215, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:22.701524] INFO [STORAGE.TRANS] print_stat_ (ob_tenant_weak_read_service.cpp:541) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=7] [WRS] [TENANT_WEAK_READ_SERVICE] [STAT](tenant_id=1002, server_version={version:{val:1719306022290605002, v:0}, total_part_count:5, valid_inner_part_count:0, valid_user_part_count:4}, server_version_delta=410918, in_cluster_service=false, cluster_version={val:18446744073709551615, v:3}, min_cluster_version={val:18446744073709551615, v:3}, max_cluster_version={val:18446744073709551615, v:3}, get_cluster_version_err=0, cluster_version_delta=-1, cluster_service_master="10.191.69.52:2882", cluster_service_tablet_id={id:226}, post_cluster_heartbeat_count=58488570, succ_cluster_heartbeat_count=57300687, cluster_heartbeat_interval=50000, local_cluster_version={val:1718276596455090001, v:0}, local_cluster_delta=1029426246433, force_self_check=false, weak_read_refresh_interval=100000) [2024-06-25 17:00:22.701578] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=19] get_max_applied_scn(scn={val:1713323697231063001, v:0}, this={ls_id_:{id:1010}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713323697231063001, v:0}}) [2024-06-25 17:00:22.701589] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=9] get_min_unreplayed_log_info(lsn={lsn:75761242098}, scn={val:1719306022290605004, v:0}, this={ls_id_:{id:1010}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306022662141, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:75761242098}, next_to_submit_scn_:{val:1719306022290605004, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713323697231063001, v:0}, iterator_:{iterator_impl:{buf_:0x2bc474493000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:75761241976}, end_lsn:{lsn:75761242098}, read_buf:{buf_len_:77824, buf_:0x2bc474493000}, block_size:67104768, log_storage_:0x2b3aaf175470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306022290605003, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306022290605003, v:0}, data_checksum:2935954383, flag:1}}, init_mode_version:0, accumulate_checksum:3603781003, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306022290605003, v:0}}}}}) [2024-06-25 17:00:22.703226] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=3] [PALF_DUMP](palf_id=1009, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1009, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:103053513, max_lsn:86651603847, max_scn:{val:1719306022641190000, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:86651603847}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:86609660807}}, last_submit_lsn:{lsn:86651603725}, last_submit_end_lsn:{lsn:86651603847}, last_submit_log_id:103053513, last_submit_log_pid:1, max_flushed_lsn:{lsn:86651603725}, max_flushed_end_lsn:{lsn:86651603847}, max_flushed_log_pid:1, committed_end_lsn:{lsn:86651603847}, last_slide_log_id:103053513, last_slide_scn:{val:1719306022641190000, v:0}, last_slide_lsn:{lsn:86651603725}, last_slide_end_lsn:{lsn:86651603847}, last_slide_log_pid:1, last_slide_log_accum_checksum:3659038166, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2b3aaf09c530}, [StateMgr]={this:0x2b3aaf0a02b0, self:"10.191.69.51:2882", palf_id:1009, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, leader:"10.186.69.176:2882", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:103053514, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x2b3aaf0a6b70, allow_vote:true}, [ConfigMgr]={palf_id:1009, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:1, config_seq:2}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1719306018901109, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1719306018901109, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2b3aaf09dc70}, [ModeMgr]={palf_id:1009, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697230007001, v:0}}, accepted_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697230007001, v:0}}, last_submit_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697230007001, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1009, is_inited:true, min_block_max_scn:{val:1718668517515990001, v:0}, min_block_id:1279, min_block_min_scn:{val:1718613154059146003, v:0}, base_lsn_for_block_gc:{lsn:86632255488}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, log_config_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, log_snapshot_meta:{version:1, base_lsn:{lsn:86632255488}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697230007001, v:0}}}, log_meta_storage:{log_tail:{lsn:2494464}, readable_log_tail:{lsn:2494464}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:86699360256}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1009}, block_mgr:{log_dir:"/data/clog/tenant_1002/1009/meta", dir_fd:824, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:64610304, block_header_serialize_buf_:0x2b3aaf0a2438, flashback_version:0}, log_storage:{log_tail:{lsn:86651603847}, readable_log_tail:{lsn:86651603847}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:86632255488}, min_scn:{val:1719289618279238004, v:0}, curr_block_id:1291, palf_id:1009}, block_mgr:{log_dir:"/data/clog/tenant_1002/1009/log", dir_fd:825, min_block_id:1279, max_block_id:1292, curr_writable_block_id:1291}, logical_block_size_:67104768, curr_block_writable_size_:47756409, block_header_serialize_buf_:0x2b3aaf0a40b8, flashback_version:0}, palf_epoch:4, last_purge_throttling_ts:-1, this:0x2b3aaf0a09f0}, [Reconfirm]={palf_id:1009, self:"10.191.69.51:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2b3aaf0a0430}) [2024-06-25 17:00:22.703337] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=109][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1009, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:22.703343] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF_DUMP](palf_id=1009, self="10.191.69.51:2882", ack_info_list=[]) [2024-06-25 17:00:22.703348] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1009, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:22.703353] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=4] current log_task status(palf_id=1009, self="10.191.69.51:2882", log_id=103053514, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[]) [2024-06-25 17:00:22.706746] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100267][T1004_L0_G0][T1004][YB420ABA45B1-00061645E8BA2BA5-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:22.711615] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100070][T1004_L0_G0][T1004][YB420ABA45B1-00061645E8BA2BA5-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:22.723113] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164323C50530-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5386, fly_ts=22301551, send_timestamp=1719306000421562) [2024-06-25 17:00:22.725037] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100030][T1004_L0_G0][T1004][YB420ABA45AB-000616C7AB4FF940-0-0] [lt=8][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:22.728148] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:22.729517] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100059][T1004_L0_G0][T1004][YB420ABA45AB-000616C7AB4FF940-0-0] [lt=8][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:22.730809] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=11000, remain_us=29300627, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=11, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.731621] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=22, this=0x2bcde0525470, extra_info="") [2024-06-25 17:00:22.731634] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b589fc96e50, extra_info="id:1039") [2024-06-25 17:00:22.731639] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:22.731667] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:22.731675] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.731677] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.731680] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.731705] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=70, this=0x2bcde052f3b0, extra_info="palf_id:1039") [2024-06-25 17:00:22.731721] INFO [LIB] stat (utility.h:1214) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=12] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbff3b0, extra_info="") [2024-06-25 17:00:22.731729] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=4, this=0x2bcde052f5e0, extra_info="palf_id:1039") [2024-06-25 17:00:22.731871] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABDC060-0006164325962B78-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301555, send_timestamp=1719306000430315) [2024-06-25 17:00:22.731906] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-0006164325962B79-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301558, send_timestamp=1719306000430348) [2024-06-25 17:00:22.731913] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B78-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2bcde05256a0, extra_info="palf_id:1039") [2024-06-25 17:00:22.731919] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B78-0-0] [lt=6] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=308, this=0x2bcde05258d0, extra_info="palf_id:1039") [2024-06-25 17:00:22.732153] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.734331] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:22.738936] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=7] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=5) [2024-06-25 17:00:22.740051] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=4][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.742017] INFO pn_ratelimit (group.c:551) [62497][IngressService][T0][Y0-0000000000000000-0-0] [lt=9] PNIO set ratelimit as 9223372036854775807 bytes/s, grp_id=2 [2024-06-25 17:00:22.747503] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=10987320, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=156, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:22.748077] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=4][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:22.748087] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:22.757498] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=1][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:228, local_retry_times:228, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:22.757510] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=0][errcode=-4012] failed to close result(close_ret=-4383, ret=-4012) [2024-06-25 17:00:22.757516] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62278][TbltTblUp6][T1001][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:22.757524] INFO [SERVER] process_final (ob_inner_sql_connection.cpp:734) [62278][TbltTblUp6][T0][YB420ABF4533-000616423CDF18B0-0-0] [lt=6] slow inner sql(last_ret=-4012, sql={ObIExecutor:, sql:"START TRANSACTION"}, process_time=30052537) [2024-06-25 17:00:22.757529] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [62278][TbltTblUp6][T0][YB420ABF4533-000616423CDF18B0-0-0] [lt=5][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"START TRANSACTION"}, aret=-4012, ret=-4012) [2024-06-25 17:00:22.757535] WDIAG [SERVER] start_transaction_inner (ob_inner_sql_connection.cpp:1101) [62278][TbltTblUp6][T0][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4012] start transaction failed(ret=-4012, tenant_id=1001, with_snap_shot=false) [2024-06-25 17:00:22.757544] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [62278][TbltTblUp6][T0][YB420ABF4533-000616423CDF18B0-0-0] [lt=3][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1001) [2024-06-25 17:00:22.757549] WDIAG [SERVER] start_transaction (ob_inner_sql_connection.cpp:1060) [62278][TbltTblUp6][T0][YB420ABF4533-000616423CDF18B0-0-0] [lt=4][errcode=-4012] start_transaction failed(ret=-4012, tenant_id=1001, with_snap_shot=false) [2024-06-25 17:00:22.757553] WDIAG [COMMON.MYSQLP] start_transaction (ob_mysql_transaction.cpp:61) [62278][TbltTblUp6][T0][YB420ABF4533-000616423CDF18B0-0-0] [lt=2][errcode=-4012] fail to start transaction(ret=-4012, tenant_id=1001, with_snapshot=false) [2024-06-25 17:00:22.757577] WDIAG [COMMON.MYSQLP] start (ob_mysql_transaction.cpp:82) [62278][TbltTblUp6][T0][YB420ABF4533-000616423CDF18B0-0-0] [lt=5][errcode=-4012] failed to start transaction(ret=-4012, tenant_id=1001, with_snapshot=false) [2024-06-25 17:00:22.757583] WDIAG [SERVER] do_batch_update_ (ob_tablet_table_updater.cpp:619) [62278][TbltTblUp6][T0][YB420ABF4533-000616423CDF18B0-0-0] [lt=5][errcode=-4012] fail to start transaction(ret=-4012, ret="OB_TIMEOUT", tenant_id=1002, meta_tenant_id=1001) [2024-06-25 17:00:22.761771] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [62160][ServerGTimer][T1004][Y0-0000000000000000-0-0] [lt=9] get_disk_usage(ret=0, capacity(MB):=2073600, used(MB):=1658818) [2024-06-25 17:00:22.761873] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325C30163-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301550, send_timestamp=1719306000460323) [2024-06-25 17:00:22.761899] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C30163-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1010, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=1342, avg_group_log_size=122) [2024-06-25 17:00:22.763826] INFO pktc_resp_cb_on_msg (pktc_resp.h:70) [62241][pnio1][T0][Y0-0000000000000000-0-0] [lt=8] PNIO resp cb not found: packet_id=1035877933 [2024-06-25 17:00:22.768395] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.768403] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.768405] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.768408] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:22.768410] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:22.768630] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-000616432547F084-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301567, send_timestamp=1719306000467062) [2024-06-25 17:00:22.775107] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.775664] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:38) [99398][T1004_TabletShe][T1004][Y0-0000000000000000-0-0] [lt=7] ====== [emptytablet] empty shell timer task ======(GC_EMPTY_TABLET_SHELL_INTERVAL=5000000) [2024-06-25 17:00:22.775677] INFO [STORAGE] runTimerTask (ob_empty_shell_task.cpp:99) [99398][T1004_TabletShe][T1004][Y0-0000000000000000-0-0] [lt=9] [emptytablet] succeed to change tablet to empty shell(ret=0, ret="OB_SUCCESS", ls_cnt=5, times=2363) [2024-06-25 17:00:22.777980] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.779123] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62245][pnio1][T1001][YB420ABF4533-00061661718F6556-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.781760] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.787238] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.787541] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.791675] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=84000, remain_us=21935501, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=84, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:22.795618] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=12000, remain_us=29235817, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=12, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.797308] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.806617] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC0B4-0-0] [lt=11] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=17, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a5d1db470, extra_info="") [2024-06-25 17:00:22.806706] INFO [LIB] stat (utility.h:1197) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=11] [PALF STAT WRITE LOG COST TIME](cur_stat_count=17, stat_interval=1000000, avg (count/cost)=76, this=0x2b3a5d1e53b0, extra_info="palf_id:1") [2024-06-25 17:00:22.806718] INFO [LIB] stat (utility.h:1214) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=17, stat_interval=1000000, avg (count/cost)=5, this=0x2b3a40ffddb0, extra_info="") [2024-06-25 17:00:22.806734] INFO [LIB] stat (utility.h:1197) [62591][T1_LogIOCb0][T1][Y0-0000000000000000-0-0] [lt=10] [PALF STAT FLUSH CB COST TIME](cur_stat_count=17, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a5d1e55e0, extra_info="palf_id:1") [2024-06-25 17:00:22.808719] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=10] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289107427328, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:22.811680] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF_DUMP](palf_id=1006, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1006, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:581409530, max_lsn:9248068442906, max_scn:{val:1719306022372701003, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:9248068442906}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:9248026499866}}, last_submit_lsn:{lsn:9248068442784}, last_submit_end_lsn:{lsn:9248068442906}, last_submit_log_id:581409530, last_submit_log_pid:1, max_flushed_lsn:{lsn:9248068442784}, max_flushed_end_lsn:{lsn:9248068442906}, max_flushed_log_pid:1, committed_end_lsn:{lsn:9248068442906}, last_slide_log_id:581409530, last_slide_scn:{val:1719306022372701003, v:0}, last_slide_lsn:{lsn:9248068442784}, last_slide_end_lsn:{lsn:9248068442906}, last_slide_log_pid:1, last_slide_log_accum_checksum:2865612740, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2b3b61c68530}, [StateMgr]={this:0x2b3b61c6c2b0, self:"10.191.69.51:2882", palf_id:1006, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, leader:"10.191.69.56:2882", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:581409531, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x2b3b61c72b70, allow_vote:true}, [ConfigMgr]={palf_id:1006, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:1, config_seq:16573}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:17233, lock_time:1716928613894143}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:16574}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17233, lock_time:1716928614383947}}, prev_log_proposal_id:1, prev_lsn:{lsn:3288062811884}, prev_mode_pid:1}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:1, config_seq:16574}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1719306019109685, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1719306019109685, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2b3b61c69c70}, [ModeMgr]={palf_id:1006, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543409019001, v:0}}, accepted_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543409019001, v:0}}, last_submit_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543409019001, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1006, is_inited:true, min_block_max_scn:{val:1718673409691462003, v:0}, min_block_id:123155, min_block_min_scn:{val:1718671611072387001, v:0}, base_lsn_for_block_gc:{lsn:8904332980224}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, log_config_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:1, config_seq:16573}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:17233, lock_time:1716928613894143}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:16574}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17233, lock_time:1716928614383947}}, prev_log_proposal_id:1, prev_lsn:{lsn:3288062811884}, prev_mode_pid:1}, log_snapshot_meta:{version:1, base_lsn:{lsn:8904332980224}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543409019001, v:0}}}, log_meta_storage:{log_tail:{lsn:69095424}, readable_log_tail:{lsn:69095424}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:9248110706688}, min_scn:{val:0, v:0}, curr_block_id:1, palf_id:1006}, block_mgr:{log_dir:"/data/clog/tenant_1004/1006/meta", dir_fd:848, min_block_id:1, max_block_id:2, curr_writable_block_id:1}, logical_block_size_:67104768, curr_block_writable_size_:65114112, block_header_serialize_buf_:0x2b3b61c6e438, flashback_version:0}, log_storage:{log_tail:{lsn:9248068442906}, readable_log_tail:{lsn:9248068442906}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:9248043601920}, min_scn:{val:1719303008647239001, v:0}, curr_block_id:137815, palf_id:1006}, block_mgr:{log_dir:"/data/clog/tenant_1004/1006/log", dir_fd:851, min_block_id:123155, max_block_id:137816, curr_writable_block_id:137815}, logical_block_size_:67104768, curr_block_writable_size_:42263782, block_header_serialize_buf_:0x2b3b61c700b8, flashback_version:0}, palf_epoch:2, last_purge_throttling_ts:1716532170051498, this:0x2b3b61c6c9f0}, [Reconfirm]={palf_id:1006, self:"10.191.69.51:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2b3b61c6c430}) [2024-06-25 17:00:22.811797] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=115][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1006, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:22.811803] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF_DUMP](palf_id=1006, self="10.191.69.51:2882", ack_info_list=[]) [2024-06-25 17:00:22.811810] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1006, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:22.811814] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=3] current log_task status(palf_id=1006, self="10.191.69.51:2882", log_id=581409531, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[]) [2024-06-25 17:00:22.815141] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.821053] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.821072] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e6c5480, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221532769, assoc_session_id:3221532769, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306022550661, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3aee2161f8, id:3221532769, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e6c5480}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:22.821522] INFO do_work (ob_rl_mgr.cpp:709) [62262][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=8] swc wakeup.(stat_period_=1000000, ready=false) [2024-06-25 17:00:22.825668] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC061-000616432B9A4D5D-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=330, fly_ts=50620698, send_timestamp=1719305972204969) [2024-06-25 17:00:22.826024] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABF4533-0006166171EF488B-0-0] [lt=5][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301554, send_timestamp=1719306000524469) [2024-06-25 17:00:22.827112] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95800][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB72-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:22.827332] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC931-0-0] [lt=10] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:22.829429] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=18995549, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=104, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:22.831679] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:22.831711] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:22.831717] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.831720] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.831722] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.831759] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1039, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:22.831909] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABDC060-0006164325962B7D-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301550, send_timestamp=1719306000530358) [2024-06-25 17:00:22.831949] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B7D-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1039, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=27, avg_log_submit_to_flush_time=87, avg_log_submit_to_slide_time=279) [2024-06-25 17:00:22.831982] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B7E-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1005, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=19, avg_log_submit_to_flush_time=88, avg_log_submit_to_slide_time=271) [2024-06-25 17:00:22.833268] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=28] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:22.834385] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:22.840107] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.845983] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABF4533-0006166172BEBED6-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301557, send_timestamp=1719306000544425) [2024-06-25 17:00:22.848918] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.850002] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABDC060-00061643246151ED-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5407, fly_ts=22301559, send_timestamp=1719306000548442) [2024-06-25 17:00:22.851935] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=11] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:22.852597] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=6096671, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=188, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:22.853744] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1791596, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=216, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:22.861292] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=13000, remain_us=29170143, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=13, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.861940] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164325C30167-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301556, send_timestamp=1719306000560383) [2024-06-25 17:00:22.862017] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.862241] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC96D-0-0] [lt=9] [PALF STAT LOG TASK TIME](palf_id=1010, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=93, avg_log_submit_to_slide_time=285) [2024-06-25 17:00:22.862249] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1010, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:22.864129] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1BBD-0-0] [lt=11] [PALF STAT COMMITTED LOG SIZE](palf_id=1009, self="10.191.69.51:2882", committed size=1098) [2024-06-25 17:00:22.864192] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1098) [2024-06-25 17:00:22.868457] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.868466] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.868469] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.868473] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:22.868476] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:22.868683] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-000616432547F088-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301553, send_timestamp=1719306000567129) [2024-06-25 17:00:22.877705] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164324713064-0-0] [lt=14][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=22301555, send_timestamp=1719306000576150) [2024-06-25 17:00:22.879908] INFO [SERVER] fill_tenant_ids_ (ob_all_virtual_tx_stat.cpp:139) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F4900-0-0] [lt=5] succeed to get tenant ids(all_tenants_=[1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:22.881280] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85B-0-0] [lt=4][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:22.887295] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.894099] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100260][T1004_L0_G0][T1004][YB420ABF4539-000616465C68FA5F-0-0] [lt=8][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:22.898522] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100254][T1004_L0_G0][T1004][YB420ABF4539-000616465C68FA5F-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:22.899644] WDIAG [SERVER] do_heartbeat_event (ob_heartbeat.cpp:265) [62788][LeaseHB][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] try reload schema failed(schema_version=1719272886792320, refresh_schema_info={schema_version:1719272886792320, tenant_id:1, sequence_id:1099511641796}, schema_ret=-4023) [2024-06-25 17:00:22.901393] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=10833431, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=157, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:22.903346] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=11] re_register_parent reason(palf_id=1009, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306018901109, last_first_register_time_us=1713323716483453, register_time_us=1719306018901109, curr_time_us=1719306022903345) [2024-06-25 17:00:22.903366] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=9] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1009, self="10.191.69.51:2882", reason=1, leader="10.186.69.176:2882", register_time_us=1719306022903355, last_submit_register_req_time_us=1719306022903355, register_parent_reason=1) [2024-06-25 17:00:22.903372] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=6] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1009, self="10.191.69.51:2882") [2024-06-25 17:00:22.911746] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=11] [PALF_DUMP](palf_id=1007, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1007, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:427126614, max_lsn:5896353987424, max_scn:{val:1719306022548511002, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:5896353987424}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:5896312044384}}, last_submit_lsn:{lsn:5896353987302}, last_submit_end_lsn:{lsn:5896353987424}, last_submit_log_id:427126614, last_submit_log_pid:1, max_flushed_lsn:{lsn:5896353987302}, max_flushed_end_lsn:{lsn:5896353987424}, max_flushed_log_pid:1, committed_end_lsn:{lsn:5896353987424}, last_slide_log_id:427126614, last_slide_scn:{val:1719306022548511002, v:0}, last_slide_lsn:{lsn:5896353987302}, last_slide_end_lsn:{lsn:5896353987424}, last_slide_log_pid:1, last_slide_log_accum_checksum:1322123509, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2b3b5b74e530}, [StateMgr]={this:0x2b3b5b7522b0, self:"10.191.69.51:2882", palf_id:1007, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, leader:"10.186.69.171:2882", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:427126615, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x2b3b5b758b70, allow_vote:true}, [ConfigMgr]={palf_id:1007, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:1, config_seq:16629}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:17204, lock_time:1716901297163077}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:16630}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17204, lock_time:1716901297438256}}, prev_log_proposal_id:1, prev_lsn:{lsn:2567935319826}, prev_mode_pid:1}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:1, config_seq:16630}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1719306020110277, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1719306020110277, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2b3b5b74fc70}, [ModeMgr]={palf_id:1007, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543410073001, v:0}}, accepted_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543410073001, v:0}}, last_submit_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543410073001, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1007, is_inited:true, min_block_max_scn:{val:1718672362983241059, v:0}, min_block_id:86098, min_block_min_scn:{val:1718670914750046005, v:0}, base_lsn_for_block_gc:{lsn:5781478391808}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, log_config_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:1, config_seq:16629}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:17204, lock_time:1716901297163077}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:16630}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17204, lock_time:1716901297438256}}, prev_log_proposal_id:1, prev_lsn:{lsn:2567935319826}, prev_mode_pid:1}, log_snapshot_meta:{version:1, base_lsn:{lsn:5781478391808}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543410073001, v:0}}}, log_meta_storage:{log_tail:{lsn:69251072}, readable_log_tail:{lsn:69251072}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:5896361754624}, min_scn:{val:0, v:0}, curr_block_id:1, palf_id:1007}, block_mgr:{log_dir:"/data/clog/tenant_1004/1007/meta", dir_fd:854, min_block_id:1, max_block_id:2, curr_writable_block_id:1}, logical_block_size_:67104768, curr_block_writable_size_:64958464, block_header_serialize_buf_:0x2b3b5b754438, flashback_version:0}, log_storage:{log_tail:{lsn:5896353987424}, readable_log_tail:{lsn:5896353987424}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:5896294649856}, min_scn:{val:1719288144384066004, v:0}, curr_block_id:87867, palf_id:1007}, block_mgr:{log_dir:"/data/clog/tenant_1004/1007/log", dir_fd:874, min_block_id:86098, max_block_id:87868, curr_writable_block_id:87867}, logical_block_size_:67104768, curr_block_writable_size_:7767200, block_header_serialize_buf_:0x2b3b5b7560b8, flashback_version:0}, palf_epoch:3, last_purge_throttling_ts:1716532096931799, this:0x2b3b5b7529f0}, [Reconfirm]={palf_id:1007, self:"10.191.69.51:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2b3b5b752430}) [2024-06-25 17:00:22.911857] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=109][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1007, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:22.911863] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF_DUMP](palf_id=1007, self="10.191.69.51:2882", ack_info_list=[]) [2024-06-25 17:00:22.911867] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=3][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1007, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:22.911871] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=3] current log_task status(palf_id=1007, self="10.191.69.51:2882", log_id=427126615, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[]) [2024-06-25 17:00:22.914753] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.915953] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=4] ====== checkpoint timer task ====== [2024-06-25 17:00:22.916448] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=7] succ to get rec scn(*this={this:0x2b3a5f004030, ls_id:{id:1}, tenant_id:1, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1719272886793317000, v:0}, last_push_gc_task_ts_:1719272888851374, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:952}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:22.916462] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=13] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a5fcac080, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1719272888852446, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a5fcac080, snapshot_version:{val:1719272888852446, v:0}, ls_id:{id:1}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:22.916477] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=11] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1}) [2024-06-25 17:00:22.916530] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=7] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3a5d156510, is_inited:true, freezing_scn:{val:1719272886793317000, v:0}, ls:{ls_meta:{tenant_id:1, ls_id:{id:1}, ls_create_status:1, clog_checkpoint_scn:{val:1719272888853692000, v:0}, clog_base_lsn:{lsn:35095793664}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:18446744073709551615, v:3}, tablet_change_checkpoint_scn:{val:1719285609440317000, v:0}, all_id_meta:{id_meta:[{limited_id:1719306035353935000, latest_log_ts:{val:1719306018786724001, v:0}}, {limited_id:80000001, latest_log_ts:{val:1719298896011036001, v:0}}, {limited_id:2660000001, latest_log_ts:{val:1719295281004326001, v:0}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a437f0030, is_in_stop_state_:false, is_inited_:true, id_:1}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:22.916561] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=30] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x2b3a5d1561b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x2b3a5d15bcd0, table_type:1, memtable_head:75, memtable_tail:76, t3m:0x2b399e3c0030, tables:[null, null, null, null, null, null, null, null, null, null, null, 0x2b3a5fc78b90, null, null, null, null]}, is_freezing:false, ls_id:{id:1}, mini_merge_recycle_commit_versions_ts:1719272889194619, tx_data_table:0x2b3a5d15d1d0, ls_tablet_svr:0x2b3a5d156190, slice_allocator:0x2b3a5d15d210}, min_rec_scn={val:1719272888853692000, v:0}, ls_id_={id:1}) [2024-06-25 17:00:22.916578] INFO [STORAGE.TRANS] get_rec_scn (ob_id_service.cpp:307) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=13] get rec log scn(service_type_=0, rec_log_ts={val:1719306018786724001, v:0}) [2024-06-25 17:00:22.916582] INFO [STORAGE.TRANS] get_rec_scn (ob_id_service.cpp:307) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=4] get rec log scn(service_type_=1, rec_log_ts={val:1719298896011036001, v:0}) [2024-06-25 17:00:22.916585] INFO [STORAGE.TRANS] get_rec_scn (ob_id_service.cpp:307) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=2] get rec log scn(service_type_=2, rec_log_ts={val:1719295281004326001, v:0}) [2024-06-25 17:00:22.916592] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=3] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1719272888853692000, v:0}, checkpoint_scn_in_ls_meta={val:1719272888853692000, v:0}, ls_id={id:1}, service_type="TRANS_SERVICE") [2024-06-25 17:00:22.916603] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=6][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:35095793664}, new_base_lsn={lsn:35095793664}, lsn={lsn:35095793664}) [2024-06-25 17:00:22.916610] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=6] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:35095793664}, ls->get_ls_id()={id:1}) [2024-06-25 17:00:22.916614] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:191) [62720][T1_TxCkpt][T1][Y0-0000000000000000-0-0] [lt=3] succeed to update_clog_checkpoint(ret=0, ls_cnt=1) [2024-06-25 17:00:22.922905] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301562, send_timestamp=1719306000621342) [2024-06-25 17:00:22.922945] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [62718][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=9] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=969588866, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=409098, mem_used=767425536) [2024-06-25 17:00:22.923677] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.926519] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.928003] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=14000, remain_us=29103433, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=14, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.929569] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=85000, remain_us=21797606, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=85, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:22.930302] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.931728] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:22.931757] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1005, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:22.931766] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:22.931768] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.931771] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.931773] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.931955] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABDC060-0006164325962B82-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301549, send_timestamp=1719306000630405) [2024-06-25 17:00:22.931963] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325962B83-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301551, send_timestamp=1719306000630412) [2024-06-25 17:00:22.934442] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:22.935634] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=12][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4013, dropped:260, tid:65259}]) [2024-06-25 17:00:22.936104] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.936113] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:22.936118] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.936125] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.936129] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.939001] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=9] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=4) [2024-06-25 17:00:22.940162] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.942880] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95800][T1004_L0_G1][T1004][YB420ABA45AB-000616C7F23FFD2E-0-0] [lt=5] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1005, child={server:"10.186.69.171:2882", region:"sys_region", register_time_us:1719306022942022, keepalive_ts:-1}) [2024-06-25 17:00:22.942888] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95800][T1004_L0_G1][T1004][YB420ABA45AB-000616C7F23FFD2E-0-0] [lt=8] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1005, self="10.191.69.51:2882", child={server:"10.186.69.171:2882", region:"sys_region", register_time_us:1719306022942022, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:22.945949] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.945958] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:22.945963] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.945969] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.945974] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.950201] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:22.955682] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [62818][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=9] current monitor number(seq_=-1) [2024-06-25 17:00:22.957580] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B85-0-0] [lt=6] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=88, avg_log_submit_to_slide_time=848) [2024-06-25 17:00:22.957590] INFO [PALF] read (log_cache.cpp:100) [99519][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=36] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:22.961198] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [42764][T1002_ComTimerT][T1002][Y0-0000000000000000-0-0] [lt=9] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-06-25 17:00:22.961982] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC060-0006164325C3016B-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301554, send_timestamp=1719306000660427) [2024-06-25 17:00:22.962008] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C3016B-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf16f470, extra_info="") [2024-06-25 17:00:22.962106] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=81, this=0x2b3aaf1793b0, extra_info="palf_id:1010") [2024-06-25 17:00:22.962117] INFO [LIB] stat (utility.h:1214) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=5, this=0x2b3a8ce1d3b0, extra_info="") [2024-06-25 17:00:22.962126] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=6, this=0x2b3aaf1795e0, extra_info="palf_id:1010") [2024-06-25 17:00:22.962295] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABF4533-00061661728FC971-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301552, send_timestamp=1719306000660742) [2024-06-25 17:00:22.962317] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC971-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3aaf16f6a0, extra_info="palf_id:1010") [2024-06-25 17:00:22.962323] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC971-0-0] [lt=6] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=290, this=0x2b3aaf16f8d0, extra_info="palf_id:1010") [2024-06-25 17:00:22.963764] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.963772] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.963777] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.963783] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.963787] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.964773] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=8] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_refresh=false, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306022964768]}) [2024-06-25 17:00:22.965106] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62246][pnio1][T0][YB420ABF4533-0006166171AF65FC-0-0] [lt=0][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:22.965116] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62246][pnio1][T0][YB420ABF4533-0006166171AF65FC-0-0] [lt=10] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:22.965132] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62246][pnio1][T0][YB420ABF4533-0006166171AF65FC-0-0] [lt=8] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306022964768]}) [2024-06-25 17:00:22.967940] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.967950] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:22.967956] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.967962] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.967968] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.968474] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [62616][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=6] ====== tenant freeze timer task ====== [2024-06-25 17:00:22.968518] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:22.968527] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:22.968529] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:22.968533] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:22.968535] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:22.968751] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-000616432547F08C-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301568, send_timestamp=1719306000667182) [2024-06-25 17:00:22.970827] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.970837] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.970843] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.970848] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.970854] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.974690] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.974697] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:22.974701] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.974707] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.974711] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.976643] INFO [STORAGE.TRANS] print_statistics (ob_xa_define.cpp:467) [63464][T1001_ObXAHbWor][T1001][Y0-0000000000000000-0-0] [lt=9] xa statistics(total_active_xa_ctx_count=0, total_standby_clearup_count=0, total_success_xa_start=0, total_failure_xa_start=0, total_success_xa_prepare=0, total_failure_xa_prepare=0, total_success_xa_1pc_commit=0, total_failure_xa_1pc_commit=0, total_success_xa_2pc_commit=0, total_failure_xa_2pc_commit=0, total_failure_xa_rollback=0, total_success_dblink_promotion=0, total_failure_dblink_promotion=0, total_success_dblink=0, total_failure_dblink=0) [2024-06-25 17:00:22.976655] INFO [STORAGE.TRANS] run1 (ob_xa_trans_heartbeat_worker.cpp:83) [63464][T1001_ObXAHbWor][T1001][Y0-0000000000000000-0-0] [lt=11] XA scheduler heartbeat task statistics(avg_time=14) [2024-06-25 17:00:22.978895] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [63468][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=10][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-06-25 17:00:22.978904] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [63468][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=7] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-06-25 17:00:22.979252] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62248][pnio1][T1001][YB420ABF4533-00061661718F6558-0-0] [lt=8][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:22.980520] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.980527] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:22.980531] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.980536] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.980540] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.983187] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=18841792, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=105, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:22.987359] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:22.988938] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62747][T1_ReplaySrv4][T1][Y0-0000000000000000-0-0] [lt=7] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:22.990314] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.990322] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:22.990327] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.990332] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.990337] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.995679] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=15000, remain_us=29035756, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=15, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:22.997637] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:22.997646] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:22.997650] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:22.997656] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:22.997661] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:22.998730] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [99192][T1004_PlanCache][T1004][Y0-0000000000000000-0-0] [lt=4] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1004) [2024-06-25 17:00:22.998742] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [99192][T1004_PlanCache][T1004][Y0-0000000000000000-0-0] [lt=10] start lib cache evict(tenant_id=1004, mem_hold=1232424960, mem_limit=36238786560, cache_obj_num=4744, cache_node_num=4297) [2024-06-25 17:00:22.998748] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [99192][T1004_PlanCache][T1004][Y0-0000000000000000-0-0] [lt=5] end lib cache evict(tenant_id=1004, cache_evict_num=0, mem_hold=1232424960, mem_limit=36238786560, cache_obj_num=4744, cache_node_num=4297) [2024-06-25 17:00:22.998754] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [99192][T1004_PlanCache][T1004][Y0-0000000000000000-0-0] [lt=5] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:23.001858] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [99192][T1004_PlanCache][T1004][Y0-0000000000000000-0-0] [lt=3] Dumping All Cache Objs(alloc_obj_list.count()=4744, alloc_obj_list=[{obj_id:51, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176332}, {obj_id:54, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303715}, {obj_id:55, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:248080}, {obj_id:56, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271604}, {obj_id:57, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216033}, {obj_id:58, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:59, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179794}, {obj_id:60, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230228}, {obj_id:61, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188271}, {obj_id:62, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196930}, {obj_id:63, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174576}, {obj_id:64, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301933}, {obj_id:65, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211398}, {obj_id:66, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224486}, {obj_id:67, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207167}, {obj_id:68, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219913}, {obj_id:69, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199095}, {obj_id:70, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199093}, {obj_id:71, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193076}, {obj_id:72, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214118}, {obj_id:73, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199546}, {obj_id:74, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199766}, {obj_id:75, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190029}, {obj_id:76, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186863}, {obj_id:77, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185594}, {obj_id:78, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200193}, {obj_id:79, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193157}, {obj_id:80, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188399}, {obj_id:81, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185761}, {obj_id:82, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:83, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185793}, {obj_id:84, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:85, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185738}, {obj_id:86, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192013}, {obj_id:87, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187261}, {obj_id:88, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183532}, {obj_id:89, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210345}, {obj_id:90, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183340}, {obj_id:91, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188170}, {obj_id:92, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183730}, {obj_id:93, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191888}, {obj_id:94, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183436}, {obj_id:95, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184586}, {obj_id:96, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188962}, {obj_id:97, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:308297}, {obj_id:98, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:251495}, {obj_id:99, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:304658}, {obj_id:100, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247858}, {obj_id:102, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183659}, {obj_id:103, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:108, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:109, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:110, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:111, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:112, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:114, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:115, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:116, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:117, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:118, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176677}, {obj_id:121, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185730}, {obj_id:122, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:123, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:124, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:166010}, {obj_id:125, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:162314}, {obj_id:126, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157829}, {obj_id:127, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:128, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175546}, {obj_id:133, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179806}, {obj_id:135, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:275362}, {obj_id:136, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218570}, {obj_id:138, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:650885}, {obj_id:139, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:140, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190714}, {obj_id:141, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223442}, {obj_id:151, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174899}, {obj_id:153, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174974}, {obj_id:154, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174973}, {obj_id:155, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174899}, {obj_id:156, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176141}, {obj_id:157, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:159, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:160, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175011}, {obj_id:161, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175011}, {obj_id:162, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175061}, {obj_id:163, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175061}, {obj_id:164, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:166, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:167, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178587}, {obj_id:168, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177355}, {obj_id:170, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:171, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181212}, {obj_id:172, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178796}, {obj_id:174, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177556}, {obj_id:175, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176287}, {obj_id:176, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176601}, {obj_id:177, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177809}, {obj_id:178, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:179, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:180, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178207}, {obj_id:181, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179455}, {obj_id:182, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176359}, {obj_id:183, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177615}, {obj_id:184, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176293}, {obj_id:185, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182472}, {obj_id:186, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178855}, {obj_id:187, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181212}, {obj_id:188, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179247}, {obj_id:189, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180179}, {obj_id:190, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174869}, {obj_id:191, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176141}, {obj_id:192, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178955}, {obj_id:193, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176751}, {obj_id:194, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182472}, {obj_id:195, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176242}, {obj_id:196, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177698}, {obj_id:197, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176601}, {obj_id:198, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177506}, {obj_id:199, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178207}, {obj_id:200, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176359}, {obj_id:209, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:160009}, {obj_id:211, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197406}, {obj_id:215, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181362}, {obj_id:223, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205804}, {obj_id:225, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197099}, {obj_id:228, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177328}, {obj_id:49386, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:316735}, {obj_id:282, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205860}, {obj_id:283, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195904}, {obj_id:285, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206516}, {obj_id:286, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:295, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:296, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:297, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187113}, {obj_id:298, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:158555}, {obj_id:301, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:302, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:303, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:304, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:305, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:306, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:308, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:310, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:312, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:314, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:316, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:318, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:320, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:322, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:327, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:329, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:334, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:337, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:340, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:342, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:344, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:346, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:348, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:350, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:354, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:357, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:360, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:363, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178166}, {obj_id:364, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176744}, {obj_id:365, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174432}, {obj_id:366, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206101}, {obj_id:367, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:369, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:370, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:373, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178480}, {obj_id:374, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187719}, {obj_id:375, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195032}, {obj_id:376, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:377, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:379, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:380, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192706}, {obj_id:383, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:253971}, {obj_id:388, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176366}, {obj_id:389, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238757}, {obj_id:390, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:567414}, {obj_id:425, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:12123}, {obj_id:426, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1069986}, {obj_id:427, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173197}, {obj_id:428, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:429, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199106}, {obj_id:430, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201883}, {obj_id:431, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197366}, {obj_id:434, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:435, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:438, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:439, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320054}, {obj_id:443, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184258}, {obj_id:447, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:269538}, {obj_id:466, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189111}, {obj_id:467, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:280230}, {obj_id:468, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:250322}, {obj_id:469, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247400}, {obj_id:470, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:252471}, {obj_id:471, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181447}, {obj_id:472, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:395411}, {obj_id:590, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:611, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:615, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206101}, {obj_id:617, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187719}, {obj_id:621, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187719}, {obj_id:622, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184003}, {obj_id:623, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195048}, {obj_id:624, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195048}, {obj_id:628, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206101}, {obj_id:629, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:630, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:631, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:642, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:412625}, {obj_id:643, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221936}, {obj_id:646, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:647, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:382131}, {obj_id:648, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:430413}, {obj_id:1192, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178616}, {obj_id:1195, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184183}, {obj_id:1197, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176695}, {obj_id:1198, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225515}, {obj_id:1200, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181195}, {obj_id:1208, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180871}, {obj_id:1209, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1214, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202835}, {obj_id:1217, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192672}, {obj_id:1219, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191511}, {obj_id:1220, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184617}, {obj_id:1224, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:169446}, {obj_id:1226, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179191}, {obj_id:1229, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177488}, {obj_id:1230, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204661}, {obj_id:1236, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198400}, {obj_id:1241, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183574}, {obj_id:1242, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177518}, {obj_id:1244, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1245, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1247, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208144}, {obj_id:1250, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191877}, {obj_id:1254, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1257, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180093}, {obj_id:1258, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178606}, {obj_id:1260, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1261, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202405}, {obj_id:1262, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202403}, {obj_id:1263, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204657}, {obj_id:1265, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203351}, {obj_id:1267, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203372}, {obj_id:1269, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205148}, {obj_id:1270, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187765}, {obj_id:1272, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204407}, {obj_id:1273, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204423}, {obj_id:1275, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205160}, {obj_id:1276, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208239}, {obj_id:1277, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191735}, {obj_id:1280, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186883}, {obj_id:1281, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:417743}, {obj_id:50524, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185067}, {obj_id:50528, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191034}, {obj_id:50539, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200251}, {obj_id:50551, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182379}, {obj_id:50556, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191508}, {obj_id:50561, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188550}, {obj_id:50573, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181193}, {obj_id:50581, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218168}, {obj_id:50582, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219985}, {obj_id:50583, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191034}, {obj_id:50584, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188525}, {obj_id:1847, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1850, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:417749}, {obj_id:5833, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:358475}, {obj_id:57432, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57433, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:166014}, {obj_id:57434, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:161312}, {obj_id:57435, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:157801}, {obj_id:57436, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175593}, {obj_id:57438, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175651}, {obj_id:57441, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182262}, {obj_id:57444, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181974}, {obj_id:57447, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181654}, {obj_id:57453, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57456, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:340129}, {obj_id:57586, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57587, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57588, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57589, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57590, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208269}, {obj_id:57591, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173021}, {obj_id:57592, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173045}, {obj_id:57597, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:263079}, {obj_id:57598, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57599, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194440}, {obj_id:57601, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:334444}, {obj_id:57603, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186311}, {obj_id:57706, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223586}, {obj_id:57709, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197503}, {obj_id:57818, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221920}, {obj_id:57820, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189379}, {obj_id:57821, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57822, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57823, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57824, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57825, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57826, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57827, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57828, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57829, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:158370}, {obj_id:57831, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175592}, {obj_id:57832, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57833, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57834, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57836, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183790}, {obj_id:57837, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194767}, {obj_id:57838, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57839, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57840, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57842, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182558}, {obj_id:57844, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57846, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184078}, {obj_id:57847, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195055}, {obj_id:57849, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184430}, {obj_id:57850, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195423}, {obj_id:57853, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57854, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57857, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185646}, {obj_id:57858, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196775}, {obj_id:57875, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57877, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182878}, {obj_id:57878, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193727}, {obj_id:57879, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57880, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57881, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57882, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57883, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57884, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57885, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57886, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57888, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57889, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57890, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57891, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:684747}, {obj_id:57892, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57893, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57894, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:158368}, {obj_id:57895, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173208}, {obj_id:57896, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173054}, {obj_id:57897, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173165}, {obj_id:57898, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173051}, {obj_id:57899, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173044}, {obj_id:57900, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57901, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:284559}, {obj_id:57911, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57912, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57913, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57914, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57916, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183198}, {obj_id:57917, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194095}, {obj_id:57919, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57921, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57922, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57923, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57924, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57925, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57926, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175614}, {obj_id:57927, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175607}, {obj_id:57928, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57930, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183486}, {obj_id:57931, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194383}, {obj_id:57932, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57933, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57934, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57935, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57936, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57939, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57947, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185038}, {obj_id:57948, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196087}, {obj_id:57949, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57950, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57951, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57952, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57953, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57955, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57956, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57957, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57958, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57959, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57961, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197259}, {obj_id:57962, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57963, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57964, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57965, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57966, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232031}, {obj_id:57967, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57968, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57969, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57970, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57979, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57988, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:57989, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58007, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175607}, {obj_id:58011, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177488}, {obj_id:58015, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58024, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238848}, {obj_id:58026, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233798}, {obj_id:58106, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233798}, {obj_id:58162, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:228704}, {obj_id:58354, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232077}, {obj_id:58358, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58361, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58362, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58371, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227015}, {obj_id:58372, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:226999}, {obj_id:58373, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:554018}, {obj_id:58375, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58376, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58377, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58378, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58379, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58380, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58383, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58386, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58387, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58389, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58394, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58399, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58400, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58401, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58409, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58411, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173106}, {obj_id:58412, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173316}, {obj_id:58415, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58418, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58421, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58424, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58427, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58428, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58429, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58430, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58431, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58432, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58443, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58452, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58453, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:58455, tenant_id:1004, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230368}, {obj_id:58458, tenant_id:10 [2024-06-25 17:00:23.003260] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [99192][T1004_PlanCache][T1004][Y0-0000000000000000-0-0] [lt=1401] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:23.006333] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5942935, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=189, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:23.007392] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1637948, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=217, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:23.008142] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.008151] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.008156] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.008162] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.008168] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.008938] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289109524480, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:23.021008] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [99999][T1004_L0_G0][T1004][YB420ABF4533-0006165FFEAFFBAE-0-0] [lt=9] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1004, table_id:11018, add_timestamp:1719306023021001}) [2024-06-25 17:00:23.021019] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF05-0-0] [lt=24] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1004, table_id=11018, vtable_type={tenant_id:1004, type:3}) [2024-06-25 17:00:23.021847] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF05-0-0] [lt=10] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1004, table_id=11018, locations=[{table_id:11018, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:23.021872] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABF4533-0006165FFEAFFBAE-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301553, send_timestamp=1719306000720318) [2024-06-25 17:00:23.021878] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF05-0-0] [lt=30] success to process renew task(task={tenant_id:1004, table_id:11018, add_timestamp:1719306023021001}, locations=[{table_id:11018, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}, {table_id:11018, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023021833, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:23.022092] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABF4533-0006165FFEAFFBAE-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301563, send_timestamp=1719306000720529) [2024-06-25 17:00:23.022283] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABF4533-0006165FFEAFFBAE-0-0] [lt=1][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620701, send_timestamp=1719305972401582) [2024-06-25 17:00:23.022763] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [62265][SerScheQueue1][T1002][YB420ABF4533-0006166046AF0DA6-0-0] [lt=0][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.022775] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62265][SerScheQueue1][T1002][YB420ABF4533-0006166046AF0DA6-0-0] [lt=0] will sleep(sleep_us=12000, remain_us=3397855, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=12, v.err_=-4383, timeout_timestamp=1719306026420630) [2024-06-25 17:00:23.026270] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100180][T1004_L0_G0][T1004][YB420ABF4539-000616469113752E-0-0] [lt=8][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:23.027630] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63474][T1001_LSArchive][T1001][YB420ABF4533-000616423F8E6725-0-0] [lt=9] gc stale ls task succ [2024-06-25 17:00:23.028046] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B86-0-0] [lt=6] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=95, avg_log_submit_to_slide_time=831) [2024-06-25 17:00:23.028054] INFO [PALF] read (log_cache.cpp:100) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=3] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:23.030648] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100068][T1004_L0_G0][T1004][YB420ABF4539-000616469113752E-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:23.031785] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:23.031820] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:23.031825] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.031827] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.031830] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.033600] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95800][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF5A5F-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:23.034481] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:23.040217] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=5][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.042096] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.042105] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.042110] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.042115] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.042121] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.044617] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.044627] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.044633] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:23.044639] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.044644] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.051090] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8100FF6D9-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=16, avg_log_batch_cnt=0, total_group_log_size=7077, avg_group_log_size=442) [2024-06-25 17:00:23.051175] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=3] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=7077) [2024-06-25 17:00:23.051937] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A0AE-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=16, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a5d1db6a0, extra_info="palf_id:1") [2024-06-25 17:00:23.051944] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A0AE-0-0] [lt=6] [PALF STAT LOG LIFE TIME](cur_stat_count=16, stat_interval=1000000, avg (count/cost)=885, this=0x2b3a5d1db8d0, extra_info="palf_id:1") [2024-06-25 17:00:23.053102] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [99392][T1004_ReqMemEvi][T1004][Y0-0000000000000000-0-0] [lt=11] sql audit evict task end(request_manager_->get_tenant_id()=1004, evict_high_mem_level=20669530113, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=8345636, mem_used=15148855296) [2024-06-25 17:00:23.053766] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63292][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=9] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7309, mem_used=12478464) [2024-06-25 17:00:23.055147] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=10679677, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=158, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:23.056879] INFO [LIB] stat (utility.h:1214) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbfec70, extra_info="") [2024-06-25 17:00:23.062334] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABF4538-00061669548FCC8D-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301556, send_timestamp=1719306000760777) [2024-06-25 17:00:23.063479] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.063489] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.063495] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:23.063502] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.063507] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.064254] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1BC5-0-0] [lt=3] [PALF STAT GROUP LOG INFO](palf_id=1009, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=9, avg_log_batch_cnt=0, total_group_log_size=1098, avg_group_log_size=122) [2024-06-25 17:00:23.064309] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=1] will sleep(sleep_us=16000, remain_us=28967127, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=16, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.065097] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F08F-0-0] [lt=10] [PALF STAT LOG TASK TIME](palf_id=1009, self="10.191.69.51:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=82, avg_log_submit_to_slide_time=840) [2024-06-25 17:00:23.065102] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT HOT CACHE HIT RATE](palf_id=1009, read_size=1098, hit_cnt=9, read_cnt=9, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:23.068379] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=86000, remain_us=21658796, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=86, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:23.068573] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.068580] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.068582] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.068585] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:23.068588] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:23.068801] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABDC060-000616432547F090-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301565, send_timestamp=1719306000767235) [2024-06-25 17:00:23.079326] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62247][pnio1][T1001][YB420ABF4533-00061661718F6559-0-0] [lt=9][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:23.087416] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.101759] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=25] get_max_applied_scn(scn={val:1719306022964341000, v:0}, this={ls_id_:{id:1006}, role_:1, proposal_id_:1, palf_committed_end_lsn_:{lsn:75469143857}, last_check_scn_:{val:1719306022964341001, v:0}, max_applied_cb_scn_:{val:1719306022964341000, v:0}}) [2024-06-25 17:00:23.101774] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1018) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=14] get_min_unreplayed_log_info in skip state(lsn={lsn:0}, scn={val:1713323697225787001, v:0}, this={ls_id_:{id:1006}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1713323710485765, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:0}, next_to_submit_scn_:{val:0, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713323697225787001, v:0}, iterator_:{iterator_impl:{buf_:null, next_round_pread_size:3678208, curr_read_pos:0, curr_read_buf_start_pos:0, curr_read_buf_end_pos:0, log_storage_:{IteratorStorage:{start_lsn:{lsn:0}, end_lsn:{lsn:0}, read_buf:{buf_len_:0, buf_:null}, block_size:67104768, log_storage_:0x2b3a95381470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:18446744073709551615, v:3}, curr_entry:{LogEntryHeader:{magic:0, version:0, log_size:-1, scn_:{val:18446744073709551615, v:3}, data_checksum:0, flag:0}}, init_mode_version:0, accumulate_checksum:-1, curr_entry_is_padding:0, padding_entry_size:0, padding_entry_scn:{val:18446744073709551615, v:3}}}}}) [2024-06-25 17:00:23.108004] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.108012] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:23.108016] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:23.108022] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.108027] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.124600] WDIAG [STORAGE.TRANS] process_ (ob_gti_rpc.h:178) [62236][pnio1][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=10][errcode=0] gti rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:23.127268] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95800][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB7E-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:23.128350] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F091-0-0] [lt=6] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=92, avg_log_submit_to_slide_time=838) [2024-06-25 17:00:23.131841] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] [PALF STAT GROUP LOG INFO](palf_id=1039, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:23.131851] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT APPEND DATA SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1220) [2024-06-25 17:00:23.131857] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:23.131887] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF STAT APPEND DATA SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1220) [2024-06-25 17:00:23.131892] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:23.131895] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.131897] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.131899] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.131967] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:23.132045] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [62481][sql_nio2][T0][Y0-0000000000000000-0-0] [lt=22] new mysql sessid created(conn.sessid_=3221583978, support_ssl=false) [2024-06-25 17:00:23.132068] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [62481][sql_nio2][T0][Y0-0000000000000000-0-0] [lt=9] sm conn init succ(conn.sessid_=3221583978, sess.client_addr_="10.191.69.51:48056") [2024-06-25 17:00:23.132069] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC060-0006164325962B8C-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301558, send_timestamp=1719306000830511) [2024-06-25 17:00:23.132076] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [62481][sql_nio2][T0][Y0-0000000000000000-0-0] [lt=5] accept one succ(*s={this:0x2b3ac6e24d70, session_id:3221583978, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:23.132111] INFO eloop_run (eloop.c:143) [62232][pnio1][T0][Y0-0000000000000000-0-0] [lt=10] PNIO [ratelimit] time: 1719306023132110, bytes: 29137868737651, bw: 0.656415 MB/s, add_ts: 1000814, add_bytes: 688861 [2024-06-25 17:00:23.132142] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164325962B8D-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301588, send_timestamp=1719306000830553) [2024-06-25 17:00:23.132277] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [62481][sql_nio2][T0][Y0-0000000000000000-0-0] [lt=13][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x2b3ac6e24d70, session_id:3221583978, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:23.132288] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [62481][sql_nio2][T0][Y0-0000000000000000-0-0] [lt=9] kill and revert session(conn.sessid_=3221583978, proxy_sessid=0, server_id=1, ret=0) [2024-06-25 17:00:23.132292] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:918) [62481][sql_nio2][T0][Y0-0000000000000000-0-0] [lt=4] sock ref clean, do destroy(*s={this:0x2b3ac6e24d70, session_id:3221583978, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:23.132300] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:223) [62481][sql_nio2][T0][Y0-0000000000000000-0-0] [lt=7] mark session id unused(conn.sessid_=3221583978) [2024-06-25 17:00:23.132313] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [62481][sql_nio2][T0][Y0-0000000000000000-0-0] [lt=12] connection close(sessid=3221583978, proxy_sessid=0, tenant_id=0, server_id=1, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=false, ret=0, trace_id=Y0-0000000000000000-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:0], disconnect_state=0) [2024-06-25 17:00:23.133680] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:23.133985] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.134003] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=17000, remain_us=28897432, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=17, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.134537] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:23.136916] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=18688062, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=106, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:23.139068] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=3) [2024-06-25 17:00:23.140272] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.141957] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62415][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.142115] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62418][RpcIO][T0][Y0-0000000000000000-0-0] [lt=4] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.142178] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62417][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.142228] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62422][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.142320] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62431][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.142449] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62426][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.142460] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62416][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.142836] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62430][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.142849] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62423][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.142979] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62425][RpcIO][T0][Y0-0000000000000000-0-0] [lt=5] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.142994] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62427][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.143014] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62421][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.143027] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62429][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.143040] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62432][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.143169] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62420][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.143321] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62419][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.143506] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62424][RpcIO][T0][Y0-0000000000000000-0-0] [lt=5] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.143598] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62428][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.150112] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF798C-0-0] [lt=10] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86d1d470, extra_info="") [2024-06-25 17:00:23.150208] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=82, this=0x2b3a86d273b0, extra_info="palf_id:1007") [2024-06-25 17:00:23.150229] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3a86d275e0, extra_info="palf_id:1007") [2024-06-25 17:00:23.150429] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F092-0-0] [lt=11] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a86d1d6a0, extra_info="palf_id:1007") [2024-06-25 17:00:23.150434] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F092-0-0] [lt=4] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=293, this=0x2b3a86d1d8d0, extra_info="palf_id:1007") [2024-06-25 17:00:23.156464] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.156474] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.156479] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.156483] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.156489] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.156858] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95800][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABF1C-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1098) [2024-06-25 17:00:23.157601] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABF4538-000616694DDF523F-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1006}}) [2024-06-25 17:00:23.157620] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABF4538-000616694DDF523F-0-0] [lt=9] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306022872809002, v:0}, arg={tenant_id:1004, ls_id:{id:1006}}, result={tenant_id:1004, ls_id:{id:1006}, cur_readable_scn:{val:1719306022872809002, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:23.159270] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.159278] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:23.159282] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:23.159287] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.159292] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.160144] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=5789124, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=190, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:23.161138] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=1484202, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=218, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:23.162091] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325C30173-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301551, send_timestamp=1719306000860540) [2024-06-25 17:00:23.162376] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABF4533-00061661728FC979-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301564, send_timestamp=1719306000860811) [2024-06-25 17:00:23.163054] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.163062] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.163066] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:23.163072] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.163076] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.168634] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.168642] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.168645] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.168647] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:23.168649] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:23.168716] INFO [LIB] stat (utility.h:1214) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=18, stat_interval=1000000, avg (count/cost)=7, this=0x2b3a8ce1cc70, extra_info="") [2024-06-25 17:00:23.168809] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.168817] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:23.168821] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:23.168827] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.168832] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.168861] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-000616432547F094-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301552, send_timestamp=1719306000867308) [2024-06-25 17:00:23.169315] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [63199][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=9] ====== tenant freeze timer task ====== [2024-06-25 17:00:23.172525] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:23.172535] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:23.173244] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:23.173252] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:23.173258] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=3][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=1999686, timeout_timestamp=1719306025172547) [2024-06-25 17:00:23.173265] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:23.173268] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:23.173274] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=5][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:23.173277] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=2][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:23.173282] WDIAG [SQL] open (ob_result_set.cpp:157) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=5][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:23.173285] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=2][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:23.173290] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=5][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}) [2024-06-25 17:00:23.173296] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=6][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.173303] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=7][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:23.173311] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=6][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:23.173315] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=4][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:23.173318] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=3][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:23.173324] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3D-0-0] [lt=3][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:23.173329] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63182][T1002_TenantInf][T1002][YB420ABF4533-000616616F6E9D3D-0-0] [lt=4][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, aret=-4013, ret=-4013) [2024-06-25 17:00:23.173333] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:23.173339] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:23.173342] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:23.173348] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] query failed(ret=-4013, conn=0x2b3ac92dd190, start=1719306023172814, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:23.173353] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:23.173358] WDIAG [SHARE] load_pure_tenant_info_ (ob_tenant_info_proxy.cpp:413) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] failed to read(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", exec_tenant_id=1001, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:23.173382] WDIAG [SHARE] load_tenant_info (ob_tenant_info_proxy.cpp:368) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] failed to load purge tenant info(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, for_update=false) [2024-06-25 17:00:23.174333] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63309][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[{id:1007}, {id:1010}, {id:1006}, {id:1008}, {id:1009}]) [2024-06-25 17:00:23.178598] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.178606] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.178611] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.178617] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.178621] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.179300] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62245][pnio1][T1001][YB420ABF4533-00061661718F655A-0-0] [lt=8][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:23.187472] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.193305] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.193315] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.193320] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.193326] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.193331] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.193343] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e6a21a0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221682245, assoc_session_id:3221682245, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306022922360, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3a97f75ff8, id:3221682245, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e6a21a0}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:23.196314] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.196321] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:23.196325] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.196330] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.196336] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.204683] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=18000, remain_us=28826752, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=18, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.206327] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC0B8-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1, self="10.191.69.51:2882", committed size=6538) [2024-06-25 17:00:23.208198] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=87000, remain_us=21518978, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=87, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:23.209079] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=10525744, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=159, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:23.209150] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289109524480, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:23.211911] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=11] re_register_parent reason(palf_id=1006, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306019109685, last_first_register_time_us=1713332560377970, register_time_us=1719306019109685, curr_time_us=1719306023211910) [2024-06-25 17:00:23.211933] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=10] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1006, self="10.191.69.51:2882", reason=1, leader="10.191.69.56:2882", register_time_us=1719306023211922, last_submit_register_req_time_us=1719306023211922, register_parent_reason=1) [2024-06-25 17:00:23.211940] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=7] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1006, self="10.191.69.51:2882") [2024-06-25 17:00:23.220013] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=5] ====== check clog disk timer task ====== [2024-06-25 17:00:23.220022] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=7] get_disk_usage(ret=0, capacity(MB):=32384, used(MB):=25883) [2024-06-25 17:00:23.220033] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [62722][T1_CKClogDisk][T1][Y0-0000000000000000-0-0] [lt=5] cannot_recycle_log_size statistics(cannot_recycle_log_size=229987168, threshold=10187125555, need_update_checkpoint_scn=false) [2024-06-25 17:00:23.227543] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC941-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86de7470, extra_info="") [2024-06-25 17:00:23.227553] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC941-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:23.227633] INFO [LIB] stat (utility.h:1197) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=73, this=0x2b3a86df13b0, extra_info="palf_id:1008") [2024-06-25 17:00:23.227646] INFO [LIB] stat (utility.h:1214) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a8ce1daf0, extra_info="") [2024-06-25 17:00:23.227661] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3a86df15e0, extra_info="palf_id:1008") [2024-06-25 17:00:23.228393] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F095-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a86de76a0, extra_info="palf_id:1008") [2024-06-25 17:00:23.228398] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F095-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=841, this=0x2b3a86de78d0, extra_info="palf_id:1008") [2024-06-25 17:00:23.230116] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.230125] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.230129] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.230135] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.230140] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.231848] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] tx gc loop thread is running(MTL_ID()=1004) [2024-06-25 17:00:23.231856] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] try gc retain ctx [2024-06-25 17:00:23.232330] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1004, *ls_ptr={ls_meta:{tenant_id:1004, ls_id:{id:1039}, ls_create_status:1, clog_checkpoint_scn:{val:1719306022464265000, v:0}, clog_base_lsn:{lsn:2818400256}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306019870754003, v:0}, tablet_change_checkpoint_scn:{val:1719301157002134002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:1, proposal_id:1, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1039}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1039, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:50335808, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-06-25 17:00:23.232398] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=30] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:23.232477] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1211) [2024-06-25 17:00:23.232631] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABDC060-0006164325962B91-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301559, send_timestamp=1719306000931071) [2024-06-25 17:00:23.232865] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1004, *ls_ptr={ls_meta:{tenant_id:1004, ls_id:{id:1005}, ls_create_status:1, clog_checkpoint_scn:{val:1718725371184341000, v:0}, clog_base_lsn:{lsn:5648141217792}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306019870754003, v:0}, tablet_change_checkpoint_scn:{val:1719301157002134002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716899681409542026, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:1, proposal_id:3, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1005}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1005, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-06-25 17:00:23.232908] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:23.232979] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=34327315, this={dio_aligned_buf:{buf_write_offset:2948, buf_padding_size:1148, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:878380814, truncate_used_ts:134488022}, log_block_size:67108864, dir_fd:845, io_fd:833}, aligned_buf_len=4096, aligned_buf="Š€€€", aligned_block_offset=34324480, buf_len=113, total_write_size_=14258, total_write_size_after_dio_=438272, ob_pwrite_used_ts=7480, count=104, each_pwrite_cost=71) [2024-06-25 17:00:23.233378] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1004, *ls_ptr={ls_meta:{tenant_id:1004, ls_id:{id:1006}, ls_create_status:1, clog_checkpoint_scn:{val:1718846540868570004, v:0}, clog_base_lsn:{lsn:8904332980224}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306019870754003, v:0}, tablet_change_checkpoint_scn:{val:1719301161935089002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716928614029017002, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1006}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1006, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-06-25 17:00:23.233396] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=18] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.233610] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF5A67-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b61c69470, extra_info="") [2024-06-25 17:00:23.233690] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=62, this=0x2b3b61c733b0, extra_info="palf_id:1006") [2024-06-25 17:00:23.233714] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3b61c735e0, extra_info="palf_id:1006") [2024-06-25 17:00:23.233858] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1004, *ls_ptr={ls_meta:{tenant_id:1004, ls_id:{id:1007}, ls_create_status:1, clog_checkpoint_scn:{val:1718726419050442000, v:0}, clog_base_lsn:{lsn:5781478391808}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306019870754003, v:0}, tablet_change_checkpoint_scn:{val:1719301167059506001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716901297164071002, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1007}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1007, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-06-25 17:00:23.233876] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=18] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.233886] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B93-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b61c696a0, extra_info="palf_id:1006") [2024-06-25 17:00:23.233892] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B93-0-0] [lt=6] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=266, this=0x2b3b61c698d0, extra_info="palf_id:1006") [2024-06-25 17:00:23.234311] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=23] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:23.234339] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1004, *ls_ptr={ls_meta:{tenant_id:1004, ls_id:{id:1008}, ls_create_status:1, clog_checkpoint_scn:{val:1719196007356667000, v:0}, clog_base_lsn:{lsn:5845831864320}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1719306019870754003, v:0}, tablet_change_checkpoint_scn:{val:1719301171837486004, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:1716901298196291003, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1008}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1008, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:7936, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-06-25 17:00:23.234356] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.234593] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:23.237860] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.237868] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:23.237872] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.237877] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.237883] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.240308] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=5][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.241564] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164319BD3E6C-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5407, fly_ts=22301555, send_timestamp=1719306000940008) [2024-06-25 17:00:23.245126] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [99399][T1004_LSArchive][T1004][YB420ABF4533-00061642492E6725-0-0] [lt=7] gc stale ls task succ [2024-06-25 17:00:23.250123] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF7990-0-0] [lt=3] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:23.250131] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF7990-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:23.252083] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62746][T1_ReplaySrv3][T1][Y0-0000000000000000-0-0] [lt=14] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:23.252095] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [62746][T1_ReplaySrv3][T1][Y0-0000000000000000-0-0] [lt=9] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1719306008942553, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1719306023251475, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-06-25 17:00:23.254329] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=11] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647489, offset=60299921}, end_cursor:ObLogCursor{file_id=100, log_id=27647490, offset=60300162}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbc8050, len:1391, log_data_len:241, seq:27647489, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:23.255409] INFO [STORAGE.TRANS] refresh_elr_tenant_config_ (ob_tx_elr_util.cpp:47) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8016-0-0] [lt=7] refresh tenant config success(tenant_id=1003, *this={last_refresh_ts:1719306023254589, can_tenant_elr:true}) [2024-06-25 17:00:23.255433] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8016-0-0] [lt=9] push gts task success(*task={:""}) [2024-06-25 17:00:23.257372] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8016-0-0] [lt=5] rpc statistics(total_trans_msg_count=2, total_batch_msg_count=0) [2024-06-25 17:00:23.257944] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=13] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647490, offset=60300162}, end_cursor:ObLogCursor{file_id=100, log_id=27647491, offset=60300411}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbd0050, len:1150, log_data_len:249, seq:27647490, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:23.262051] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647491, offset=60300411}, end_cursor:ObLogCursor{file_id=100, log_id=27647492, offset=60300660}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbdc050, len:901, log_data_len:249, seq:27647491, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:23.262147] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-0006164325C30177-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301553, send_timestamp=1719306000960593) [2024-06-25 17:00:23.262153] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647492, offset=60300660}, end_cursor:ObLogCursor{file_id=100, log_id=27647493, offset=60300909}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbd8050, len:652, log_data_len:249, seq:27647492, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:23.262256] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=4] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647493, offset=60300909}, end_cursor:ObLogCursor{file_id=100, log_id=27647494, offset=60301158}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbe8050, len:403, log_data_len:249, seq:27647493, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:23.262476] INFO eloop_run (eloop.c:143) [62250][pnio2][T0][Y0-0000000000000000-0-0] [lt=11] PNIO [ratelimit] time: 1719306023262475, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007614, add_bytes: 0 [2024-06-25 17:00:23.268689] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.268696] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.268699] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.268701] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:23.268703] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:23.268921] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABDC060-000616432547F098-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301552, send_timestamp=1719306000967369) [2024-06-25 17:00:23.271944] INFO [STORAGE.TRANS] dump_mapper_info (ob_lock_wait_mgr.h:66) [100449][T1003_LockWaitM][T1003][Y0-0000000000000000-0-0] [lt=8] report RowHolderMapper summary info(count=0, bkt_cnt=248) [2024-06-25 17:00:23.276299] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=19000, remain_us=28755137, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=19, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.279396] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [63377][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=9] ====== tenant freeze timer task ====== [2024-06-25 17:00:23.279410] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [63377][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=8][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-06-25 17:00:23.279443] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62235][pnio1][T1001][YB420ABF4533-00061661718F655B-0-0] [lt=7][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:23.286463] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95800][T1004_L0_G1][T1004][YB420ABA45B0-00061644F985525D-0-0] [lt=2] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1005, child={server:"10.186.69.176:2882", region:"sys_region", register_time_us:1719306023286372, keepalive_ts:-1}) [2024-06-25 17:00:23.286473] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95800][T1004_L0_G1][T1004][YB420ABA45B0-00061644F985525D-0-0] [lt=10] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1005, self="10.191.69.51:2882", child={server:"10.186.69.176:2882", region:"sys_region", register_time_us:1719306023286372, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:23.287525] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.290537] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=18534441, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=107, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:23.292336] WDIAG [RPC] send (ob_poc_rpc_proxy.h:156) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=0][errcode=-4012] sync rpc execute fail(ret=-4012, addr="10.191.69.52:2882", pcode=1315) [2024-06-25 17:00:23.292350] WDIAG [SQL.ENG] query_statistics (ob_px_tenant_target_monitor.cpp:275) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=10][errcode=-4012] send rpc to query statistics failed, need send refresh all(ret=-4012) [2024-06-25 17:00:23.292358] WDIAG [SQL.ENG] refresh_statistics (ob_px_tenant_target_monitor.cpp:114) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=7][errcode=-4012] query statistics failed(ret=-4012) [2024-06-25 17:00:23.292364] WDIAG [COMMON] operator() (ob_px_target_mgr.h:158) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] target monitor refresh statistics failed(ret=-4012, px_tenant_info={v:1002}, px_res_info->get_target_monitor()={is_init:true, tenant_id:1002, server:"10.191.69.51:2882", dummy_cache_leader:"10.191.69.52:2882", role:2}) [2024-06-25 17:00:23.295141] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 0.93, rt: 91.00) [2024-06-25 17:00:23.295151] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SYS](tenant_id=500, io_status=sys_group_name: INSPECT_BAD_BLOCK_IO, mode: read, size: 2097152.00, iops: 0.93, rt: 763.00) [2024-06-25 17:00:23.295157] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:23.295221] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=16] [IO STATUS SYS](tenant_id=1004, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 10.24, rt: 73.27) [2024-06-25 17:00:23.295226] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=1004, ref_cnt=2, io_config={group_num_:0, memory_limit_:724775731200, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:100}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=12763552, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:23.295238] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=10] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295245] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295248] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295251] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295256] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295259] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295262] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295266] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295270] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295273] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295276] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295281] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295283] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295286] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295290] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295295] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:23.295807] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.295816] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.295821] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.295827] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.295832] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.304429] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=1][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.304439] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.304444] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:23.304450] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.304455] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.307326] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.307336] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.307341] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:23.307347] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.307352] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.311107] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.311116] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.311120] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:23.311125] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.311131] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.313803] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5635464, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=191, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:23.314882] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1330458, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=219, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:23.316931] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.316940] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.316944] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.316950] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.316955] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.326816] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.326825] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.326829] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.326835] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.326838] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.327389] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB86-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b74f470, extra_info="") [2024-06-25 17:00:23.327500] INFO [LIB] stat (utility.h:1197) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=79, this=0x2b3b5b7593b0, extra_info="palf_id:1007") [2024-06-25 17:00:23.327508] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:23.327518] INFO [LIB] stat (utility.h:1214) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3afcbffaf0, extra_info="") [2024-06-25 17:00:23.327533] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=9, this=0x2b3b5b7595e0, extra_info="palf_id:1007") [2024-06-25 17:00:23.328209] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B95-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b5b74f6a0, extra_info="palf_id:1007") [2024-06-25 17:00:23.328215] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B95-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=847, this=0x2b3b5b74f8d0, extra_info="palf_id:1007") [2024-06-25 17:00:23.328636] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABF4533-0006166171EF4890-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301553, send_timestamp=1719306001027082) [2024-06-25 17:00:23.331926] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:23.331961] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:23.331965] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.331968] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.331971] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.332052] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1005, self="10.191.69.51:2882", committed size=1211) [2024-06-25 17:00:23.332174] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164325962B96-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301558, send_timestamp=1719306001030616) [2024-06-25 17:00:23.332186] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325962B97-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301554, send_timestamp=1719306001030631) [2024-06-25 17:00:23.334613] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:23.335368] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B98-0-0] [lt=2] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=77, avg_log_submit_to_slide_time=265) [2024-06-25 17:00:23.335376] INFO [PALF] read (log_cache.cpp:100) [99515][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=13] [PALF STAT HOT CACHE HIT RATE](palf_id=1006, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:23.335989] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=11][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4383, dropped:925, tid:62275}]) [2024-06-25 17:00:23.339097] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.339107] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.339112] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.339119] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.339124] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.339133] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=7] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=2) [2024-06-25 17:00:23.340365] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.340702] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95800][T1004_L0_G1][T1004][YB420ABF4538-000616694DAF5335-0-0] [lt=7] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1005, child={server:"10.191.69.56:2882", region:"sys_region", register_time_us:1719306023340797, keepalive_ts:-1}) [2024-06-25 17:00:23.340709] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95800][T1004_L0_G1][T1004][YB420ABF4538-000616694DAF5335-0-0] [lt=7] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1005, self="10.191.69.51:2882", child={server:"10.191.69.56:2882", region:"sys_region", register_time_us:1719306023340797, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:23.341948] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.341956] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:23.341960] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.341966] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.341970] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.343879] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63467][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7381, mem_used=12478464) [2024-06-25 17:00:23.344609] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.344618] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.344622] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.344627] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.344633] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.348911] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:23.348920] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:23.348924] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb219aea0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023294539, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.348949] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=25][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb219aea0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023294539, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.348962] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999990, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:23.348969] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:23.348974] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:23.348978] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:23.348982] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:23.348986] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:23.348992] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=20, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.348999] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=6] will sleep(sleep_us=20000, remain_us=28682436, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=20, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.349060] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:23.349068] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:23.349072] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6d6bf4a0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023294539, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.349091] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=19][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6d6bf4a0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023294539, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.349103] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:23.349109] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:23.349112] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:23.349115] WDIAG [SQL] open (ob_result_set.cpp:157) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:23.349118] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:23.349123] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:23.349129] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=88, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.349133] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4] will sleep(sleep_us=88000, remain_us=21378042, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=88, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:23.349825] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.349833] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.349838] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.349843] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.349848] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.359646] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.359654] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:23.359658] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.359664] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.359668] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.362218] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-0006164325C3017B-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301564, send_timestamp=1719306001060653) [2024-06-25 17:00:23.362520] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABF4533-00061661728FC981-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301566, send_timestamp=1719306001060953) [2024-06-25 17:00:23.362842] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62456][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.362917] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62459][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.362974] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:23.362984] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:23.362988] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6935e9e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023308237, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.363006] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6935e9e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023308237, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.363017] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:23.363024] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=6][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:23.363027] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:23.363031] WDIAG [SQL] open (ob_result_set.cpp:157) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:23.363033] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:23.363037] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:23.363041] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=160, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.363047] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=10371777, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=160, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:23.363153] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62460][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.363406] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62454][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.363462] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62458][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.363592] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62457][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.363779] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62455][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.364410] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1BD1-0-0] [lt=3] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf09d470, extra_info="") [2024-06-25 17:00:23.364504] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=70, this=0x2b3aaf0a73b0, extra_info="palf_id:1009") [2024-06-25 17:00:23.364525] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=5, this=0x2b3aaf0a75e0, extra_info="palf_id:1009") [2024-06-25 17:00:23.364942] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [62725][T1_LSArchiveMgr][T1][YB420ABF4533-000616423A1E6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:23.365279] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F09B-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf09d6a0, extra_info="palf_id:1009") [2024-06-25 17:00:23.365284] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F09B-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=841, this=0x2b3aaf09d8d0, extra_info="palf_id:1009") [2024-06-25 17:00:23.366687] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62470][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.366710] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62466][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.366996] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62468][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367019] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62462][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367221] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62441][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367241] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62463][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367246] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62450][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367308] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62464][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367335] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62443][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367358] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62461][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367389] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62467][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367392] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62444][BatchIO][T0][Y0-0000000000000000-0-0] [lt=4] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367456] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62440][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367501] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62447][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367520] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62449][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367572] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62438][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367607] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62448][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367608] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62434][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367647] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62437][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367679] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62446][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367701] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62452][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367822] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62435][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367893] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62465][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.367907] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62445][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.368156] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62436][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.368243] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62451][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.368276] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62442][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.368294] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62469][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.368306] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62473][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.368323] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62471][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=8] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.368464] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62439][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.368711] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:104) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] tx gc loop thread is running(MTL_ID()=1002) [2024-06-25 17:00:23.368718] INFO [STORAGE.TRANS] run1 (ob_tx_loop_worker.cpp:111) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] try gc retain ctx [2024-06-25 17:00:23.369059] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:20, local_retry_times:20, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:23.369068] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=7][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.369072] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.369076] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:23.369082] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:23.369185] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1006}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018752321001, v:0}, clog_base_lsn:{lsn:75425759232}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315392841000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:1, proposal_id:1, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1006}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1006, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:110, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-06-25 17:00:23.369252] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=26] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.369307] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=43388965, this={dio_aligned_buf:{buf_write_offset:150, buf_padding_size:3946, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:47361090, truncate_used_ts:13648119}, log_block_size:67108864, dir_fd:816, io_fd:832}, aligned_buf_len=4096, aligned_buf="", aligned_block_offset=43388928, buf_len=113, total_write_size_=11328, total_write_size_after_dio_=393216, ob_pwrite_used_ts=6542, count=93, each_pwrite_cost=70) [2024-06-25 17:00:23.369505] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-000616432547F09C-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301558, send_timestamp=1719306001067947) [2024-06-25 17:00:23.369720] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=5] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1007}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018870127000, v:0}, clog_base_lsn:{lsn:85155950592}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315408583001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1007}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1007, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-06-25 17:00:23.369741] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=20] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.370205] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1008}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018543922003, v:0}, clog_base_lsn:{lsn:74083663872}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315055094004, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1008}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1008, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-550728704, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-06-25 17:00:23.370222] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=17] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.370696] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1009}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018805948000, v:0}, clog_base_lsn:{lsn:86632255488}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315224596003, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1009}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1009, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-06-25 17:00:23.370715] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=18] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:23.371175] INFO [STORAGE.TRANS] do_tx_gc_ (ob_tx_loop_worker.cpp:232) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] [Tx Loop Worker] check tx scheduler success(MTL_ID()=1002, *ls_ptr={ls_meta:{tenant_id:1002, ls_id:{id:1010}, ls_create_status:1, clog_checkpoint_scn:{val:1719198894656674000, v:0}, clog_base_lsn:{lsn:75627073536}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315483366000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1010}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1010, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}) [2024-06-25 17:00:23.371192] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=16] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:23.373064] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62475][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:23.376933] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100010][T1004_L0_G0][T1004][YB420ABF4538-00061669368F31CB-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:23.377497] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.377505] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:23.377510] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.377515] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.377521] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.379446] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62242][pnio1][T1001][YB420ABF4533-00061661718F655C-0-0] [lt=7][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:23.380758] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [62735][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=7] start do ls ha handler(ls_id_array_=[{id:1}]) [2024-06-25 17:00:23.380873] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100255][T1004_L0_G0][T1004][YB420ABF4538-00061669368F31CB-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:23.387580] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.388800] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABF4533-0006166172BEBEE5-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301553, send_timestamp=1719306001087246) [2024-06-25 17:00:23.390605] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=9][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:107, local_retry_times:107, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:23.390618] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=12][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.390623] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.390625] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:23.390633] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:23.391835] INFO [STORAGE.TRANS] print_statistics (ob_xa_define.cpp:467) [63289][T1002_ObXAHbWor][T1002][Y0-0000000000000000-0-0] [lt=11] xa statistics(total_active_xa_ctx_count=0, total_standby_clearup_count=0, total_success_xa_start=0, total_failure_xa_start=0, total_success_xa_prepare=0, total_failure_xa_prepare=0, total_success_xa_1pc_commit=0, total_failure_xa_1pc_commit=0, total_success_xa_2pc_commit=0, total_failure_xa_2pc_commit=0, total_failure_xa_rollback=0, total_success_dblink_promotion=0, total_failure_dblink_promotion=0, total_success_dblink=0, total_failure_dblink=0) [2024-06-25 17:00:23.396831] INFO [SHARE] force_set_master_rs (ob_rs_mgr.cpp:277) [62782][T1_L0_G0][T1][YB420ABA45AB-000616C80E7FB3F1-0-0] [lt=3] [RS_MGR] force set rs list(master_rs="10.186.69.171:2882") [2024-06-25 17:00:23.396843] INFO [SERVER] handle_heartbeat (ob_service.cpp:3094) [62782][T1_L0_G0][T1][YB420ABA45AB-000616C80E7FB3F1-0-0] [lt=9] handle_heartbeat(ret=0, ret="OB_SUCCESS", hb_request={server:"10.191.69.51:2882", server_id:1, rs_addr:"10.186.69.171:2882", rs_server_status:1, epoch_id:1}, hb_response={zone:"zone1", server:"10.191.69.51:2882", sql_port:2881, build_version:"4.2.2.0_100010012024022719-c984fe7cb7a4cef85a40323a0d073f0c9b7b8235(Feb 27 2024 19:20:54)", start_service_time:1713323604978888, server_health_status:{data_disk_status_:1, data_disk_status:"NORMAL"}}, time_cost=14) [2024-06-25 17:00:23.404672] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1961) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(dag_cnt=0, map_size=0) [2024-06-25 17:00:23.404679] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1971) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(running_dag_net_map_size=0, blocking_dag_net_list_size=0) [2024-06-25 17:00:23.404683] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_COMPACTION_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:23.404691] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(priority="PRIO_HA_HIGH", low_limit=8, up_limit=8, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:23.404694] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_COMPACTION_MID", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:23.404697] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_HA_MID", low_limit=5, up_limit=5, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:23.404701] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_COMPACTION_LOW", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:23.404704] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_HA_LOW", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:23.404707] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_DDL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:23.404709] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(priority="PRIO_DDL_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:23.404713] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_TTL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:23.404717] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:23.404721] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:23.404725] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:23.404729] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:23.404732] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:23.404735] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:23.404738] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:23.404742] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:23.404745] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:23.404748] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:23.404751] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:23.404755] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:23.404759] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:23.404761] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:23.404764] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:23.404766] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:23.404771] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:23.404774] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:23.404776] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:23.404779] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:23.404783] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404786] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404789] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404792] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404795] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404799] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404802] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404805] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404807] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404810] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404813] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404817] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404820] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404823] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404826] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404830] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404833] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404836] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404839] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404843] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404845] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404848] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404851] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404856] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404858] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404861] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404863] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404868] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404870] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:23.404873] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404876] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-06-25 17:00:23.404880] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-06-25 17:00:23.404883] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-06-25 17:00:23.404886] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-06-25 17:00:23.404888] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:23.404893] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:23.404895] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:23.404898] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:23.404901] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:23.404906] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:23.404908] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:23.404911] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:23.404914] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:23.404919] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:23.404921] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:23.404924] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:23.404927] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:23.404931] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:23.404933] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:23.404936] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:23.404939] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:23.404944] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:23.404946] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:23.404949] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404952] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:23.404957] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404959] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:23.404962] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404966] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:23.404970] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404972] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:23.404976] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404979] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:23.404982] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404985] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:23.404990] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:23.404993] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:23.405000] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=7] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:23.405002] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:23.405005] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:23.405008] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:23.405012] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:23.405015] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, dag_count=0) [2024-06-25 17:00:23.405017] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, scheduled_task_count=0) [2024-06-25 17:00:23.405020] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, dag_count=0) [2024-06-25 17:00:23.405024] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, scheduled_task_count=0) [2024-06-25 17:00:23.405027] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, dag_count=0) [2024-06-25 17:00:23.405030] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-06-25 17:00:23.405032] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, dag_count=0) [2024-06-25 17:00:23.405036] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-06-25 17:00:23.405039] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, dag_count=0) [2024-06-25 17:00:23.405043] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, scheduled_task_count=0) [2024-06-25 17:00:23.405045] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_MIGRATION", dag_net_count=0) [2024-06-25 17:00:23.405049] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_PREPARE_MIGRATION", dag_net_count=0) [2024-06-25 17:00:23.405051] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_COMPLETE_MIGRATION", dag_net_count=0) [2024-06-25 17:00:23.405054] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER", dag_net_count=0) [2024-06-25 17:00:23.405056] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_BACKUP", dag_net_count=0) [2024-06-25 17:00:23.405059] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_RESTORE", dag_net_count=0) [2024-06-25 17:00:23.405062] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TYPE_BACKUP_CLEAN", dag_net_count=0) [2024-06-25 17:00:23.405064] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER_BACKFILL_TX", dag_net_count=0) [2024-06-25 17:00:23.405067] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1991) [63251][T1002_DagSchedu][T1002][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(total_worker_cnt=43, total_running_task_cnt=0, work_thread_num=43, scheduled_task_cnt=0) [2024-06-25 17:00:23.407281] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A0B1-0-0] [lt=10] [PALF STAT LOG TASK TIME](palf_id=1, self="10.191.69.51:2882", total_slide_log_cnt=15, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=91, avg_log_submit_to_slide_time=890) [2024-06-25 17:00:23.407290] INFO [PALF] read (log_cache.cpp:100) [62748][T1_ReplaySrv5][T1][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1, read_size=6538, hit_cnt=14, read_cnt=14, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:23.409391] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289084846080, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:23.411350] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=2][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.411359] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.411364] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.411370] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.411375] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.413867] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=9][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:191, local_retry_times:191, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:23.413880] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=11][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.413883] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.413887] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:23.413894] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:23.414946] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=8][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:219, local_retry_times:219, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:23.414956] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.414959] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.414961] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:23.414967] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:23.416164] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=5] ====== check clog disk timer task ====== [2024-06-25 17:00:23.416177] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=7] get_disk_usage(ret=0, capacity(MB):=5529, used(MB):=4371) [2024-06-25 17:00:23.416192] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [63296][T1002_CKClogDis][T1002][Y0-0000000000000000-0-0] [lt=5] cannot_recycle_log_size statistics(cannot_recycle_log_size=288403628, threshold=1739461755, need_update_checkpoint_scn=false) [2024-06-25 17:00:23.422679] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:23.422689] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:23.422693] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb2193820, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023368114, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.422717] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=24][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb2193820, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023368114, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.422730] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999990, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:23.422738] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:23.422743] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:23.422746] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:23.422750] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:23.422753] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:23.422759] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=21, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.422765] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4] will sleep(sleep_us=21000, remain_us=28608671, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=21, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.425686] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.425696] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.425702] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:23.425709] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.425714] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.425720] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:23.425725] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:23.425730] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:23.425733] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:23.425737] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:23.425742] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:23.425745] WDIAG [SQL.ENG] inner_get_next_batch_for_tsc (ob_table_scan_op.cpp:2015) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:23.425748] WDIAG [SQL.ENG] inner_get_next_batch (ob_table_scan_op.cpp:2000) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:23.425752] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] get next batch failed(ret=-4383, eval_ctx={batch_idx:0, batch_size:16, max_batch_size:16, frames_:0x2b5cd7214380}, id=2, op_name="PHY_TABLE_SCAN") [2024-06-25 17:00:23.425760] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1535) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4383] get next batch failed(ret=-4383) [2024-06-25 17:00:23.425763] WDIAG [SQL.ENG] get_next_row_vectorizely (ob_operator.cpp:1463) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] Failed to get_next_row vectorizedly(ret=-4383) [2024-06-25 17:00:23.425767] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_transmit_op.cpp:27) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] get next row from child failed(ret=-4383) [2024-06-25 17:00:23.425771] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] inner get next row failed(ret=-4383, type=14, op="PHY_DIRECT_TRANSMIT", op_id=1) [2024-06-25 17:00:23.425777] WDIAG [SQL.EXE] sync_send_result (ob_remote_executor_processor.cpp:283) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:23.425782] WDIAG [SQL.EXE] execute_remote_plan (ob_remote_executor_processor.cpp:469) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] sync send result failed(ret=-4383) [2024-06-25 17:00:23.425787] WDIAG [SQL.EXE] process (ob_remote_executor_processor.cpp:969) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=2][errcode=-4383] execute remote plan failed(ret=-4383, local_tenant_schema_version=1718209168713032, exec_ctx_.get_task_exec_ctx().get_query_tenant_begin_schema_version()=1718209168713032) [2024-06-25 17:00:23.428264] INFO [LIB] stat (utility.h:1197) [99521][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=12] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=7, this=0x2b3ae6e6c8f8, extra_info="") [2024-06-25 17:00:23.428274] INFO [LIB] stat (utility.h:1197) [99521][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=1, this=0x2b3ae6e6cb28, extra_info="") [2024-06-25 17:00:23.430674] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95800][T1004_L0_G1][T1004][YB420ABA45AD-0006164650E9E671-0-0] [lt=6] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1039, child={server:"10.186.69.173:2882", region:"sys_region", register_time_us:1719306023431574, keepalive_ts:-1}) [2024-06-25 17:00:23.430682] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95800][T1004_L0_G1][T1004][YB420ABA45AD-0006164650E9E671-0-0] [lt=8] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1039, self="10.191.69.51:2882", child={server:"10.186.69.173:2882", region:"sys_region", register_time_us:1719306023431574, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:23.431980] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:23.432013] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:23.432018] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.432020] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.432023] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.432036] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164325210E44-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=330, fly_ts=22301554, send_timestamp=1719306001130481) [2024-06-25 17:00:23.432207] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325962B9B-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301551, send_timestamp=1719306001130655) [2024-06-25 17:00:23.434667] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:23.435933] INFO [STORAGE.BLKMGR] inspect_bad_block (ob_block_manager.cpp:1712) [62325][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=12] check macro block(block_info={ref_cnt:1, is_free:false, access_time:1717064079546539}, time_interval=2241943889392) [2024-06-25 17:00:23.437200] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:88, local_retry_times:88, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:23.437211] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.437214] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.437218] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:23.437223] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:23.440421] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.442429] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.442439] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.442444] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:23.442451] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.442456] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.443825] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=7][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:21, local_retry_times:21, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:23.443834] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.443836] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.443839] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:23.443846] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:23.444334] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:23.444344] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:23.444348] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a67905ce0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023390222, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.444368] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=20][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a67905ce0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023390222, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.444381] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:23.444387] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:23.444391] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:23.444395] WDIAG [SQL] open (ob_result_set.cpp:157) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:23.444398] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:23.444402] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:23.444406] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=108, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.444413] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=6] will sleep(sleep_us=100000, remain_us=18380566, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=108, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:23.452223] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62746][T1_ReplaySrv3][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:23.457018] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABF28-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b6d5470, extra_info="") [2024-06-25 17:00:23.457105] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=14] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=71, this=0x2b3b5b6df3b0, extra_info="palf_id:1008") [2024-06-25 17:00:23.457128] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3b5b6df5e0, extra_info="palf_id:1008") [2024-06-25 17:00:23.457867] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B9E-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b5b6d56a0, extra_info="palf_id:1008") [2024-06-25 17:00:23.457872] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962B9E-0-0] [lt=4] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=849, this=0x2b3b5b6d58d0, extra_info="palf_id:1008") [2024-06-25 17:00:23.463108] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:160, local_retry_times:160, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:23.463118] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.463121] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.463124] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:23.463130] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:23.464174] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [62538][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=8] dump tenant info(tenant={id:1, tenant_meta:{unit:{tenant_id:1, unit_id:1, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1, name:"sys_unit_config", resource:{min_cpu:6, max_cpu:6, memory_size:"8GB", log_disk_size:"31.625GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:4}}, mode:0, create_timestamp:1713323559851672, is_removed:false}, super_block:{tenant_id:1, replay_start_point:ObLogCursor{file_id=1, log_id=301825, offset=35151636}, ls_meta_entry:[1183703](ver=0,mode=0,seq=14642313), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:false, version:2}, create_status:1}, unit_min_cpu:"6.000000000000000000e+00", unit_max_cpu:"6.000000000000000000e+00", total_worker_cnt:89, min_worker_cnt:62, max_worker_cnt:150, stopped:0, worker_us:526103408043, recv_hp_rpc_cnt:4052014, recv_np_rpc_cnt:5922166, recv_lp_rpc_cnt:0, recv_mysql_cnt:51386558, recv_task_cnt:3387908, recv_large_req_cnt:0, tt_large_quries:315062, pop_normal_cnt:24058080473, workers:62, nesting workers:8, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=574 cnt[3]=0 cnt[4]=0 cnt[5]=40451529 cnt[6]=0 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:group_id = 2,queue_size = 0,recv_req_cnt = 11964924,min_worker_cnt = 6,max_worker_cnt = 150,worker_cnt = 6,token_change = 1719206890267396 group_id = 10,queue_size = 0,recv_req_cnt = 59766051,min_worker_cnt = 6,max_worker_cnt = 150,worker_cnt = 6,token_change = 1719206890267669 group_id = 15,queue_size = 0,recv_req_cnt = 11456,min_worker_cnt = 2,max_worker_cnt = 2,worker_cnt = 0,token_change = 1719303225511892 group_id = 9,queue_size = 0,recv_req_cnt = 64442023,min_worker_cnt = 6,max_worker_cnt = 150,worker_cnt = 6,token_change = 1719206890267872 group_id = 1,queue_size = 0,recv_req_cnt = 150207250,min_worker_cnt = 8,max_worker_cnt = 8,worker_cnt = 1,token_change = 1719305976625126 , rpc_stat_info: pcode=0x1501:cnt=116 pcode=0x1516:cnt=114 pcode=0x14a:cnt=107 pcode=0x51f:cnt=101 pcode=0x4a9:cnt=100, token_change_ts:1719206890268087, tenant_role:1}) [2024-06-25 17:00:23.464696] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325C3017F-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301555, send_timestamp=1719306001163140) [2024-06-25 17:00:23.464808] INFO [PALF] inner_write_once_ (log_block_handler.cpp:411) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT WRITE LOG INFO TO DISK](ret=0, offset=67068744, this={dio_aligned_buf:{buf_write_offset:953, buf_padding_size:3143, align_size:4096, aligned_buf_size:3682304, aligned_used_ts:48241997, truncate_used_ts:13719527}, log_block_size:67108864, dir_fd:828, io_fd:844}, aligned_buf_len=4096, aligned_buf="", aligned_block_offset=67067904, buf_len=113, total_write_size_=12182, total_write_size_after_dio_=421888, ob_pwrite_used_ts=7655, count=100, each_pwrite_cost=76) [2024-06-25 17:00:23.464548] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [62538][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=373] dump tenant info(tenant={id:508, tenant_meta:{unit:{tenant_id:508, unit_id:1000, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1000, name:"virtual_tenant_unit", resource:{min_cpu:5, max_cpu:5, memory_size:"1GB", log_disk_size:"2GB", min_iops:10000, max_iops:50000, iops_weight:0}}, mode:0, create_timestamp:1713323553487523, is_removed:false}, super_block:{tenant_id:508, replay_start_point:ObLogCursor{file_id=1, log_id=1, offset=0}, ls_meta_entry:[-1](ver=0,mode=0,seq=0), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:true, version:2}, create_status:1}, unit_min_cpu:"5.000000000000000000e+00", unit_max_cpu:"5.000000000000000000e+00", total_worker_cnt:52, min_worker_cnt:52, max_worker_cnt:150, stopped:0, worker_us:20218841301, recv_hp_rpc_cnt:0, recv_np_rpc_cnt:315653196, recv_lp_rpc_cnt:0, recv_mysql_cnt:0, recv_task_cnt:0, recv_large_req_cnt:0, tt_large_quries:0, pop_normal_cnt:30357366540, workers:52, nesting workers:0, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=0 cnt[3]=0 cnt[4]=0 cnt[5]=0 cnt[6]=0 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:group_id = 15,queue_size = 0,recv_req_cnt = 79,min_worker_cnt = 2,max_worker_cnt = 2,worker_cnt = 0,token_change = 1719221347488094 , rpc_stat_info: pcode=0x5c0:cnt=342, token_change_ts:1713323610935409, tenant_role:0}) [2024-06-25 17:00:23.464993] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABF4533-00061661728FC986-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301557, send_timestamp=1719306001163435) [2024-06-25 17:00:23.464886] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [62538][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=338] dump tenant info(tenant={id:1001, tenant_meta:{unit:{tenant_id:1001, unit_id:1008, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1001, name:"ocp_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"0.6GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1}}, mode:0, create_timestamp:1713323654823784, is_removed:false}, super_block:{tenant_id:1001, replay_start_point:ObLogCursor{file_id=1, log_id=1, offset=0}, ls_meta_entry:[-1](ver=0,mode=0,seq=0), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:false, version:2}, create_status:1}, unit_min_cpu:"1.000000000000000000e+00", unit_max_cpu:"1.000000000000000000e+00", total_worker_cnt:7, min_worker_cnt:6, max_worker_cnt:150, stopped:0, worker_us:2976240531, recv_hp_rpc_cnt:24784458, recv_np_rpc_cnt:3571436, recv_lp_rpc_cnt:0, recv_mysql_cnt:0, recv_task_cnt:0, recv_large_req_cnt:0, tt_large_quries:0, pop_normal_cnt:2423553193, workers:6, nesting workers:0, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=0 cnt[3]=0 cnt[4]=0 cnt[5]=0 cnt[6]=0 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:group_id = 5,queue_size = 0,recv_req_cnt = 15781,min_worker_cnt = 1,max_worker_cnt = 150,worker_cnt = 1,token_change = 1717592211283158 , rpc_stat_info: pcode=0x743:cnt=1, token_change_ts:1713323655172487, tenant_role:1}) [2024-06-25 17:00:23.465263] INFO [STORAGE.TRANS] statistics (ob_gts_source.cpp:70) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=11] gts statistics(tenant_id=1, gts_rpc_cnt=10, get_gts_cache_cnt=150, get_gts_with_stc_cnt=0, try_get_gts_cache_cnt=0, try_get_gts_with_stc_cnt=0, wait_gts_elapse_cnt=0, try_wait_gts_elapse_cnt=0) [2024-06-25 17:00:23.465274] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=10] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1, need_refresh=false, gts_local_cache={srr:[mts=1719306022964757], gts:1719306022964056000, latest_srr:[mts=1719306023465250]}) [2024-06-25 17:00:23.465283] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=8] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-06-25 17:00:23.465223] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [62538][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=337] dump tenant info(tenant={id:1002, tenant_meta:{unit:{tenant_id:1002, unit_id:1008, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1001, name:"ocp_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"1GB", log_disk_size:"5.4GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1}}, mode:0, create_timestamp:1713323654823784, is_removed:false}, super_block:{tenant_id:1002, replay_start_point:ObLogCursor{file_id=65, log_id=18131796, offset=18975273}, ls_meta_entry:[964242](ver=0,mode=0,seq=6194880), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:false, version:2}, create_status:1}, unit_min_cpu:"1.000000000000000000e+00", unit_max_cpu:"1.000000000000000000e+00", total_worker_cnt:18, min_worker_cnt:6, max_worker_cnt:150, stopped:0, worker_us:4611900884470, recv_hp_rpc_cnt:449961946, recv_np_rpc_cnt:99167407, recv_lp_rpc_cnt:0, recv_mysql_cnt:297519435, recv_task_cnt:15540, recv_large_req_cnt:0, tt_large_quries:479041, pop_normal_cnt:3533162097, workers:6, nesting workers:8, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=0 cnt[3]=0 cnt[4]=0 cnt[5]=391 cnt[6]=0 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:group_id = 2,queue_size = 0,recv_req_cnt = 119646243,min_worker_cnt = 1,max_worker_cnt = 150,worker_cnt = 1,token_change = 1713323708321768 group_id = 15,queue_size = 0,recv_req_cnt = 2399,min_worker_cnt = 2,max_worker_cnt = 2,worker_cnt = 0,token_change = 1718273625504891 group_id = 5,queue_size = 0,recv_req_cnt = 3905,min_worker_cnt = 1,max_worker_cnt = 150,worker_cnt = 1,token_change = 1718390909881694 group_id = 19,queue_size = 0,recv_req_cnt = 2391053,min_worker_cnt = 1,max_worker_cnt = 150,worker_cnt = 1,token_change = 1719305964358932 group_id = 100,queue_size = 0,recv_req_cnt = 4342800,min_worker_cnt = 8,max_worker_cnt = 150,worker_cnt = 0,token_change = 1719305984903423 group_id = 1,queue_size = 0,recv_req_cnt = 1185716193,min_worker_cnt = 8,max_worker_cnt = 8,worker_cnt = 1,token_change = 1719305989107944 , rpc_stat_info: pcode=0x1516:cnt=391 pcode=0x1501:cnt=391 pcode=0x1502:cnt=372 pcode=0x150a:cnt=100 pcode=0x150b:cnt=100, token_change_ts:1719294457568407, tenant_role:1}) [2024-06-25 17:00:23.465628] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62241][pnio1][T0][YB420ABF4533-0006166171AF6601-0-0] [lt=8][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:23.465637] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62241][pnio1][T0][YB420ABF4533-0006166171AF6601-0-0] [lt=9] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:23.465650] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62241][pnio1][T0][YB420ABF4533-0006166171AF6601-0-0] [lt=6] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306023465292]}) [2024-06-25 17:00:23.465598] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [62538][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=374] dump tenant info(tenant={id:1003, tenant_meta:{unit:{tenant_id:1003, unit_id:1022, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1003, name:"u1", resource:{min_cpu:10, max_cpu:10, memory_size:"75GB", log_disk_size:"225GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:100}}, mode:0, create_timestamp:1713332511418312, is_removed:false}, super_block:{tenant_id:1003, replay_start_point:ObLogCursor{file_id=1, log_id=1, offset=0}, ls_meta_entry:[-1](ver=0,mode=0,seq=0), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:false, version:2}, create_status:1}, unit_min_cpu:"1.000000000000000000e+01", unit_max_cpu:"1.000000000000000000e+01", total_worker_cnt:42, min_worker_cnt:42, max_worker_cnt:960, stopped:0, worker_us:14260083155, recv_hp_rpc_cnt:26299008, recv_np_rpc_cnt:5865857, recv_lp_rpc_cnt:0, recv_mysql_cnt:0, recv_task_cnt:0, recv_large_req_cnt:0, tt_large_quries:0, pop_normal_cnt:23814594665, workers:42, nesting workers:0, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=0 cnt[3]=0 cnt[4]=0 cnt[5]=0 cnt[6]=0 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:, rpc_stat_info: pcode=0x743:cnt=28 pcode=0x651:cnt=20 pcode=0x745:cnt=8, token_change_ts:1713332512024862, tenant_role:1}) [2024-06-25 17:00:23.465931] INFO [SERVER.OMT] run1 (ob_multi_tenant.cpp:2335) [62538][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=333] dump tenant info(tenant={id:1004, tenant_meta:{unit:{tenant_id:1004, unit_id:1022, has_memstore:true, unit_status:"NORMAL", config:{unit_config_id:1003, name:"u1", resource:{min_cpu:100, max_cpu:100, memory_size:"675GB", log_disk_size:"2025GB", min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:100}}, mode:0, create_timestamp:1713332511418312, is_removed:false}, super_block:{tenant_id:1004, replay_start_point:ObLogCursor{file_id=56, log_id=15626764, offset=52786127}, ls_meta_entry:[1204639](ver=0,mode=0,seq=6188816), tablet_meta_entry:[-1](ver=0,mode=0,seq=0), is_hidden:false, version:2}, create_status:1}, unit_min_cpu:"1.000000000000000000e+02", unit_max_cpu:"1.000000000000000000e+02", total_worker_cnt:512, min_worker_cnt:402, max_worker_cnt:8640, stopped:0, worker_us:2443281372750, recv_hp_rpc_cnt:153636955, recv_np_rpc_cnt:212576106, recv_lp_rpc_cnt:0, recv_mysql_cnt:6079016702, recv_task_cnt:161544, recv_large_req_cnt:0, tt_large_quries:91668744, pop_normal_cnt:249553559300, workers:402, nesting workers:8, req_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 , multi_level_queue:total_size=0 queue[0]=0 queue[1]=0 queue[2]=0 queue[3]=0 queue[4]=0 queue[5]=0 queue[6]=0 queue[7]=0 queue[8]=0 queue[9]=0 , recv_level_rpc_cnt:cnt[0]=0 cnt[1]=0 cnt[2]=222 cnt[3]=0 cnt[4]=0 cnt[5]=9528 cnt[6]=1948 cnt[7]=0 cnt[8]=0 cnt[9]=0 , group_map:group_id = 2,queue_size = 0,recv_req_cnt = 136101120,min_worker_cnt = 100,max_worker_cnt = 8640,worker_cnt = 100,token_change = 1713332554345183 group_id = 15,queue_size = 0,recv_req_cnt = 2713,min_worker_cnt = 2,max_worker_cnt = 2,worker_cnt = 0,token_change = 1719303255566407 group_id = 19,queue_size = 0,recv_req_cnt = 3315529,min_worker_cnt = 100,max_worker_cnt = 8640,worker_cnt = 1,token_change = 1719306022182661 group_id = 100,queue_size = 0,recv_req_cnt = 49707,min_worker_cnt = 100,max_worker_cnt = 8640,worker_cnt = 0,token_change = 1718738388256531 group_id = 16,queue_size = 0,recv_req_cnt = 97,min_worker_cnt = 100,max_worker_cnt = 8640,worker_cnt = 0,token_change = 1716928639383969 group_id = 1,queue_size = 0,recv_req_cnt = 3853536373,min_worker_cnt = 100,max_worker_cnt = 100,worker_cnt = 1,token_change = 1719306022126766 , rpc_stat_info: pcode=0x1502:cnt=820 pcode=0x1501:cnt=305 pcode=0x1516:cnt=305 pcode=0x150b:cnt=200 pcode=0x51f:cnt=150, token_change_ts:1713332511737338, tenant_role:1}) [2024-06-25 17:00:23.467535] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:23.467545] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:23.467550] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e7796e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023413318, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.467569] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=18][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e7796e0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023413318, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.467581] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:23.467587] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:23.467593] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:23.467596] WDIAG [SQL] open (ob_result_set.cpp:157) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:23.467601] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:23.467605] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:23.467611] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=192, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.467608] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABA45B0-00061644F03C36DE-0-0] [lt=12] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1008}}) [2024-06-25 17:00:23.467616] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4] will sleep(sleep_us=100000, remain_us=5481651, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=192, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:23.467623] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABA45B0-00061644F03C36DE-0-0] [lt=7] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306023142141003, v:0}, arg={tenant_id:1004, ls_id:{id:1008}}, result={tenant_id:1004, ls_id:{id:1008}, cur_readable_scn:{val:1719306023142141003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:23.468643] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:23.468653] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:23.468657] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6ab3cfe0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023414372, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.468675] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=18][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6ab3cfe0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023414372, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.468687] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999998, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:23.468692] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:23.468697] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:23.468700] WDIAG [SQL] open (ob_result_set.cpp:157) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:23.468703] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:23.468707] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}) [2024-06-25 17:00:23.468713] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, retry_cnt=220, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.468718] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4] will sleep(sleep_us=100000, remain_us=1176623, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=220, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:23.468805] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.468813] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.468816] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.468819] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:23.468822] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:23.472479] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABA45AB-000616C80A1FC7F9-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1007}}) [2024-06-25 17:00:23.472490] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABA45AB-000616C80A1FC7F9-0-0] [lt=5] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306023048488003, v:0}, arg={tenant_id:1004, ls_id:{id:1007}}, result={tenant_id:1004, ls_id:{id:1007}, cur_readable_scn:{val:1719306023048488003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:23.477252] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.477260] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.477265] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:23.477271] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.477275] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.479541] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62238][pnio1][T1001][YB420ABF4533-00061661718F655D-0-0] [lt=8][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:23.486825] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [100471][T1003_LSArchive][T1003][YB420ABF4533-0006164249CE6725-0-0] [lt=7] gc stale ls task succ [2024-06-25 17:00:23.487425] INFO [PALF] runTimerTask (block_gc_timer_task.cpp:101) [63175][T1002_PalfGC][T1002][Y0-0000000000000000-0-0] [lt=17] BlockGCTimerTask success(ret=0, cost_time_us=5, palf_env_impl_={IPalfEnvImpl:{IPalfEnvImpl:"Dummy"}, self:"10.191.69.51:2882", log_dir:"/data/clog/tenant_1002", disk_options_wrapper:{disk_opts_for_stopping_writing:{log_disk_size(MB):5529, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):5529, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):595, sequence:1}, log_alloc_mgr_:{flying_log_task:0, flying_meta_task:0}}) [2024-06-25 17:00:23.487636] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.491028] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:23.491037] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:23.491040] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6d6b6080, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023436380, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.491058] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=18][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6d6b6080, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023436380, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.491069] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=10][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:23.491075] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:23.491080] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:23.491082] WDIAG [SQL] open (ob_result_set.cpp:157) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:23.491086] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:23.491089] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:23.491094] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=89, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.491100] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5] will sleep(sleep_us=89000, remain_us=21236076, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=89, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:23.493742] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [100481][T1003_HAService][T1003][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:23.497461] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:23.497472] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:23.497476] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb2175e20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023443698, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.497493] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb2175e20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023443698, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.497505] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999990, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:23.497512] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:23.497516] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:23.497518] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:23.497522] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:23.497526] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:23.497529] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=22, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.497536] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=6] will sleep(sleep_us=22000, remain_us=28533899, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=22, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.500004] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [99409][T1004_HAService][T1004][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[{id:1039}, {id:1008}, {id:1007}, {id:1006}, {id:1005}]) [2024-06-25 17:00:23.516897] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:23.516906] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:23.516909] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6935cc40, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023462573, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.516924] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=15][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6935cc40, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023462573, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.516935] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=10][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:23.516940] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:23.516943] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:23.516946] WDIAG [SQL] open (ob_result_set.cpp:157) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:23.516949] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:23.516952] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:23.516956] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=161, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.516960] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3] will sleep(sleep_us=100000, remain_us=10217863, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=161, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:23.519592] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:22, local_retry_times:22, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:23.519603] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.519607] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.519609] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:23.519615] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:23.520180] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [63375][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=3] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1001) [2024-06-25 17:00:23.520192] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [63375][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=10] start lib cache evict(tenant_id=1001, mem_hold=48234496, mem_limit=53687090, cache_obj_num=160, cache_node_num=160) [2024-06-25 17:00:23.520199] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [63375][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=7] end lib cache evict(tenant_id=1001, cache_evict_num=0, mem_hold=48234496, mem_limit=53687090, cache_obj_num=160, cache_node_num=160) [2024-06-25 17:00:23.520205] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [63375][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=5] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:23.522280] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [63375][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=4] Dumping All Cache Objs(alloc_obj_list.count()=160, alloc_obj_list=[{obj_id:74, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176269}, {obj_id:170, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179296}, {obj_id:2925, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2927, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:2930, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:3104, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238685}, {obj_id:3670, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198516}, {obj_id:4047, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199084}, {obj_id:4236, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187574}, {obj_id:4312, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189568}, {obj_id:4410, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189807}, {obj_id:4422, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4423, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4424, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190714}, {obj_id:4425, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178773}, {obj_id:4426, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176478}, {obj_id:4427, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177216}, {obj_id:4428, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:4429, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:4430, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196176}, {obj_id:4431, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177512}, {obj_id:4432, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:4433, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189999}, {obj_id:4434, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190072}, {obj_id:4435, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198144}, {obj_id:4436, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187766}, {obj_id:4439, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176429}, {obj_id:4440, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4441, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:4442, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4443, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:4444, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:4445, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4446, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188501}, {obj_id:4448, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188033}, {obj_id:4449, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186593}, {obj_id:4451, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184744}, {obj_id:4452, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180056}, {obj_id:4453, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181280}, {obj_id:4454, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4455, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231854}, {obj_id:4456, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188880}, {obj_id:4457, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195808}, {obj_id:4458, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177512}, {obj_id:4459, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231828}, {obj_id:4460, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227365}, {obj_id:4461, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4462, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188501}, {obj_id:4464, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4465, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188514}, {obj_id:4466, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234073}, {obj_id:4467, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230619}, {obj_id:4468, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238569}, {obj_id:4469, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237412}, {obj_id:4470, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240809}, {obj_id:4471, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:240814}, {obj_id:4472, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177206}, {obj_id:4477, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185193}, {obj_id:4479, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189690}, {obj_id:4483, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4484, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214740}, {obj_id:4485, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242121}, {obj_id:4486, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203713}, {obj_id:4487, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189157}, {obj_id:4488, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197503}, {obj_id:4490, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:450404}, {obj_id:4491, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4494, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:12123}, {obj_id:4495, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1069986}, {obj_id:4496, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173197}, {obj_id:4497, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199106}, {obj_id:4498, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201883}, {obj_id:4499, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197366}, {obj_id:4501, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183800}, {obj_id:4502, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223549}, {obj_id:4504, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231160}, {obj_id:4505, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222320}, {obj_id:4506, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4507, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4508, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4511, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4512, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4513, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4514, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4515, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4516, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4517, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:340122}, {obj_id:4518, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209292}, {obj_id:4519, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232400}, {obj_id:4520, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188656}, {obj_id:4522, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187041}, {obj_id:4523, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:239894}, {obj_id:4524, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189541}, {obj_id:4525, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223651}, {obj_id:4527, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:293345}, {obj_id:4528, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260848}, {obj_id:4529, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:264414}, {obj_id:4530, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176744}, {obj_id:4531, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174432}, {obj_id:4532, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177149}, {obj_id:4533, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181222}, {obj_id:4534, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179110}, {obj_id:4535, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176843}, {obj_id:4536, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:300030}, {obj_id:4537, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190162}, {obj_id:4540, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242569}, {obj_id:4541, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206423}, {obj_id:4542, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:461761}, {obj_id:4543, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:347873}, {obj_id:4544, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174698}, {obj_id:4545, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256486}, {obj_id:4546, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176308}, {obj_id:4547, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:269328}, {obj_id:4548, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241975}, {obj_id:4549, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195211}, {obj_id:4550, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4551, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4552, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197431}, {obj_id:4553, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194857}, {obj_id:4554, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4555, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222198}, {obj_id:4556, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187377}, {obj_id:4557, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193422}, {obj_id:4559, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:323398}, {obj_id:4560, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4562, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185688}, {obj_id:4565, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180645}, {obj_id:4566, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:243065}, {obj_id:4567, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:244210}, {obj_id:4568, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:306729}, {obj_id:4569, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4570, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207409}, {obj_id:4571, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180749}, {obj_id:4572, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180381}, {obj_id:4573, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199212}, {obj_id:4574, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:236313}, {obj_id:4575, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234022}, {obj_id:4576, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207409}, {obj_id:4577, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:283358}, {obj_id:4578, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190463}, {obj_id:4579, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191548}, {obj_id:4580, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190949}, {obj_id:4581, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192036}, {obj_id:4582, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203777}, {obj_id:4583, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222644}, {obj_id:4584, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194701}, {obj_id:4585, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206398}, {obj_id:4586, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4587, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185048}, {obj_id:4588, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:4589, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247578}, {obj_id:4590, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:251025}, {obj_id:4591, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225724}, {obj_id:4592, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:261613}, {obj_id:4593, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179688}, {obj_id:4594, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180912}, {obj_id:4595, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276666}, {obj_id:4596, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179635}, {obj_id:4597, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179232}, {obj_id:4598, tenant_id:1001, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:279990}]) [2024-06-25 17:00:23.522433] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [63375][T1001_PlanCache][T1001][Y0-0000000000000000-0-0] [lt=152] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:23.525444] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=5] ====== check clog disk timer task ====== [2024-06-25 17:00:23.525452] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=7] get_disk_usage(ret=0, capacity(MB):=230400, used(MB):=0) [2024-06-25 17:00:23.525459] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [100468][T1003_CKClogDis][T1003][Y0-0000000000000000-0-0] [lt=4] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=72477573120, need_update_checkpoint_scn=false) [2024-06-25 17:00:23.527799] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:23.532025] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:23.532052] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=16, this=0x2b3b5a817470, extra_info="") [2024-06-25 17:00:23.532061] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5a884e50, extra_info="id:1005") [2024-06-25 17:00:23.532068] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:23.532071] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.532073] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.532076] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.532146] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=69, this=0x2b3b5a8213b0, extra_info="palf_id:1005") [2024-06-25 17:00:23.532165] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8215e0, extra_info="palf_id:1005") [2024-06-25 17:00:23.532252] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164325962BA0-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301556, send_timestamp=1719306001230695) [2024-06-25 17:00:23.532320] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BA1-0-0] [lt=3] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8176a0, extra_info="palf_id:1005") [2024-06-25 17:00:23.532328] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BA1-0-0] [lt=8] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=289, this=0x2b3b5a8178d0, extra_info="palf_id:1005") [2024-06-25 17:00:23.533647] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95800][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF5A73-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=1211) [2024-06-25 17:00:23.534344] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.534354] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.534359] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.534366] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.534371] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.534721] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:23.536197] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [100464][T1003_ReqMemEvi][T1003][Y0-0000000000000000-0-0] [lt=10] sql audit evict task end(request_manager_->get_tenant_id()=1003, evict_high_mem_level=1932735283, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=83036, mem_used=195495936) [2024-06-25 17:00:23.537207] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.537216] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.537221] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:23.537227] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.537237] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=10][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.539097] INFO [DETECT] record_summary_info_and_logout_when_necessary_ (ob_lcl_batch_sender_thread.cpp:203) [63459][T1001_LCLSender][T1001][Y0-0000000000000000-0-0] [lt=13] ObLCLBatchSenderThread periodic report summary info(duty_ratio_percentage=0, total_constructed_detector=30358025, total_destructed_detector=30358025, total_alived_detector=0, _lcl_op_interval=30000, lcl_msg_map_.count()=0, *this={this:0x2b3a995e42b0, is_inited:true, is_running:true, total_record_time:5010000, over_night_times:0}) [2024-06-25 17:00:23.539198] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=9] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=1) [2024-06-25 17:00:23.540474] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.540707] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC061-00061643952B83C0-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=50620703, send_timestamp=1719305972920004) [2024-06-25 17:00:23.541013] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.541022] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.541026] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=2][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:23.541032] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.541037] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.544475] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=6][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:108, local_retry_times:108, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:23.544485] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.544489] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.544492] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:23.544500] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:23.546788] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.546798] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.550410] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0A2-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=94, avg_log_submit_to_slide_time=285) [2024-06-25 17:00:23.550416] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:23.556568] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.557176] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:23.560331] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F4902-0-0] [lt=9][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=1204, pos=316, bitmap=NULL) [2024-06-25 17:00:23.560512] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F4902-0-0] [lt=10][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=9, len=131, pos=98, bitmap=NULL) [2024-06-25 17:00:23.560702] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F4902-0-0] [lt=5][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=394, pos=303, bitmap=NULL) [2024-06-25 17:00:23.560865] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F4902-0-0] [lt=5][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=1, len=38, pos=17, bitmap=NULL) [2024-06-25 17:00:23.562324] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-0006164325C30183-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301567, send_timestamp=1719306001260756) [2024-06-25 17:00:23.562616] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABF4533-00061661728FC989-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301563, send_timestamp=1719306001261052) [2024-06-25 17:00:23.562656] INFO [LIB] stat (utility.h:1197) [63319][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=3] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=40, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a872048f8, extra_info="") [2024-06-25 17:00:23.562666] INFO [LIB] stat (utility.h:1197) [63319][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=40, stat_interval=1000000, avg (count/cost)=1, this=0x2b3a87204b28, extra_info="") [2024-06-25 17:00:23.567678] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=6][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:192, local_retry_times:192, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:23.567688] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.567691] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.567694] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:23.567699] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:23.568779] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:220, local_retry_times:220, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:23.568789] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.568792] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:23.568795] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:23.568802] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:23.568869] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.568877] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.568880] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.568882] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:23.568884] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:23.569096] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-000616432547F0A4-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301558, send_timestamp=1719306001267537) [2024-06-25 17:00:23.572396] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.572407] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.572415] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:23.572420] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:23.572426] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:23.572429] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:23.572433] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:23.572435] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:23.572439] WDIAG [SQL.ENG] inner_get_next_batch_for_tsc (ob_table_scan_op.cpp:2015) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:23.572443] WDIAG [SQL.ENG] inner_get_next_batch (ob_table_scan_op.cpp:2000) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:23.572446] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] get next batch failed(ret=-4383, eval_ctx={batch_idx:0, batch_size:16, max_batch_size:16, frames_:0x2b5bd0e13580}, id=2, op_name="PHY_TABLE_SCAN") [2024-06-25 17:00:23.572454] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1535) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7][errcode=-4383] get next batch failed(ret=-4383) [2024-06-25 17:00:23.572456] WDIAG [SQL.ENG] get_next_row_vectorizely (ob_operator.cpp:1463) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4383] Failed to get_next_row vectorizedly(ret=-4383) [2024-06-25 17:00:23.572462] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_transmit_op.cpp:27) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4383] get next row from child failed(ret=-4383) [2024-06-25 17:00:23.572465] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2][errcode=-4383] inner get next row failed(ret=-4383, type=14, op="PHY_DIRECT_TRANSMIT", op_id=1) [2024-06-25 17:00:23.572468] WDIAG [SQL.EXE] sync_send_result (ob_remote_executor_processor.cpp:283) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:23.572471] WDIAG [SQL.EXE] execute_remote_plan (ob_remote_executor_processor.cpp:469) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4383] sync send result failed(ret=-4383) [2024-06-25 17:00:23.572475] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=2] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e6ba2c0, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221546856, assoc_session_id:3221546856, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306023304024, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3aee23fbf8, id:3221546856, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e6ba2c0}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:23.572509] WDIAG [SQL.EXE] process (ob_remote_executor_processor.cpp:969) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=30][errcode=-4383] execute remote plan failed(ret=-4383, local_tenant_schema_version=1718209168713032, exec_ctx_.get_task_exec_ctx().get_query_tenant_begin_schema_version()=1718209168713032) [2024-06-25 17:00:23.573183] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:23.573192] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:23.573196] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb2179960, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306023519403, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:23.573218] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=23000, remain_us=28458217, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=23, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.574496] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.574839] INFO [SQL.EXE] run2 (ob_maintain_dependency_info_task.cpp:221) [62306][MaintainDepInfo][T0][Y0-0000000000000000-0-0] [lt=7] [ASYNC TASK QUEUE](queue_size=0) [2024-06-25 17:00:23.579543] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62235][pnio1][T1001][YB420ABF4533-00061661718F655E-0-0] [lt=7][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.580168] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.581307] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62747][T1_ReplaySrv4][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:23.581317] INFO [STORAGE] ~ObStorageTableGuard (ob_storage_table_guard.cpp:156) [62747][T1_ReplaySrv4][T1][Y0-0000000000000000-0-0] [lt=8] throttle statics(need_speed_limit=false, last_throttle_status=false, last_print_log_time=1719306013046005, stat={total_throttle_time_us:0, total_skip_throttle_time_us:0, last_log_timestamp:1719306023580423, last_throttle_status:false, 0=0, 1=0, 2=0, 3=0}) [2024-06-25 17:00:23.587697] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.598101] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.598117] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=18226861, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=109, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:23.607082] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.607093] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.609628] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=14] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289107427328, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:23.613236] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [62703][T1_LSMetaCh][T1][YB420ABF4533-0006166172AE6CE6-0-0] [lt=47] fetch root ls success(local_cluster_id=1713323324, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.171:2882", sql_port:2881, role:1, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607828518, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.176:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:4, zone:"zone4", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607829360, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.189.192.96:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:5, zone:"zone5", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607830236, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.51:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607831113, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.56:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["10.186.69.171:2882", "10.191.69.51:2882", "10.191.69.56:2882", "10.186.69.176:2882", "10.189.192.96:2882"]) [2024-06-25 17:00:23.617019] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.618127] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63299][T1002_LSArchive][T1002][YB420ABF4533-000616423EEE6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:23.618506] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63484][T1001_HAService][T1001][Y0-0000000000000000-0-0] [lt=7] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:23.621249] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5328019, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=193, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:23.622433] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.622450] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=1022890, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=221, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:23.624910] INFO [COMMON] clean_garbage_node (ob_kvcache_map.cpp:646) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=14] Cache wash clean map node details(ret=0, clean_node_count=8, clean_time=0, clean_start_pos=173543443, clean_num=402653) [2024-06-25 17:00:23.632078] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:23.632111] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:23.632116] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.632119] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.632121] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.632292] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325962BA5-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301552, send_timestamp=1719306001330739) [2024-06-25 17:00:23.632355] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-0006164325962BA6-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301555, send_timestamp=1719306001330799) [2024-06-25 17:00:23.633992] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=90000, remain_us=21093183, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=90, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:23.634776] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:23.635319] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=22] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:23.640518] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=6][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.650005] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=24000, remain_us=28381430, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=24, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.652310] INFO [LIB] stat (utility.h:1197) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=9] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a3f4048f8, extra_info="") [2024-06-25 17:00:23.652320] INFO [LIB] stat (utility.h:1197) [62750][T1_ReplaySrv7][T1][Y0-0000000000000000-0-0] [lt=10] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=54, this=0x2b3a3f404b28, extra_info="") [2024-06-25 17:00:23.657128] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95800][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABF30-0-0] [lt=6] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:23.662371] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164325C30187-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301556, send_timestamp=1719306001360815) [2024-06-25 17:00:23.662401] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C30187-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1010, self="10.191.69.51:2882", committed size=1089) [2024-06-25 17:00:23.662488] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=15] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1010, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1211) [2024-06-25 17:00:23.662662] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABF4533-00061661728FC98D-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301565, send_timestamp=1719306001361097) [2024-06-25 17:00:23.668531] INFO [SERVER] do_batch_update_ (ob_tablet_table_updater.cpp:645) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=5] REPORT: batch update tablets finished(ret=-4012, ret="OB_TIMEOUT", replicas.count()=2, cost_time=32059982) [2024-06-25 17:00:23.668541] WDIAG [SERVER] batch_process_tasks (ob_tablet_table_updater.cpp:514) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=9][errcode=-4012] do_batch_update_ failed(tmp_ret=-4012, tmp_ret="OB_TIMEOUT", start_time=1719305991608515, tasks count=2, tablet replicas count=2) [2024-06-25 17:00:23.668548] INFO [SERVER] dump (ob_tablet_table_updater.cpp:66) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=6] [TABLET_TABLE_UPDATE_STATISTIC] dump tablet table update statistics(suc_cnt=0, fail_cnt=2, remove_task_cnt=0, update_task_cnt=2, avg_wait_us=1012269479007, avg_exec_us=16030016) [2024-06-25 17:00:23.668559] WDIAG [SERVER] batch_process_tasks (ob_uniq_task_queue.h:508) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=6][errcode=-4012] fail to batch process task(ret=-4012) [2024-06-25 17:00:23.668564] WDIAG [SERVER] run1 (ob_uniq_task_queue.h:459) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A7-0-0] [lt=5][errcode=-4012] fail to batch execute task(ret=-4012, tasks.count()=2) [2024-06-25 17:00:23.668579] WDIAG [SERVER] check_task_status (ob_tablet_table_updater.cpp:150) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4016] tablet table update task cost too much time to execute(*this={tenant_id:1002, ls_id:{id:1009}, tablet_id:{id:204769}, add_timestamp:1718298873893038}, safe_interval=7200000000, cost_time=1007149775541) [2024-06-25 17:00:23.668602] WDIAG [SERVER] check_task_status (ob_tablet_table_updater.cpp:150) [62272][TbltTblUp0][T0][YB420ABF4533-000616423D1F16A8-0-0] [lt=7][errcode=-4016] tablet table update task cost too much time to execute(*this={tenant_id:1002, ls_id:{id:1007}, tablet_id:{id:204717}, add_timestamp:1718288570365977}, safe_interval=7200000000, cost_time=1017453302625) [2024-06-25 17:00:23.668929] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=21, this=0x2b3a9537b470, extra_info="") [2024-06-25 17:00:23.668943] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:23.668951] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT APPEND DATA SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1211) [2024-06-25 17:00:23.668957] INFO [LIB] stat (utility.h:1197) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a9534ee50, extra_info="id:1006") [2024-06-25 17:00:23.668961] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.668966] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.668968] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.668971] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:23.668974] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:23.669013] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=15] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=70, this=0x2b3a953853b0, extra_info="palf_id:1006") [2024-06-25 17:00:23.669023] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1211) [2024-06-25 17:00:23.669038] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1006, self="10.191.69.51:2882", committed size=1211) [2024-06-25 17:00:23.669047] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=4, this=0x2b3a953855e0, extra_info="palf_id:1006") [2024-06-25 17:00:23.669179] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-000616432547F0A8-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301552, send_timestamp=1719306001367627) [2024-06-25 17:00:23.669236] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0A8-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a9537b6a0, extra_info="palf_id:1006") [2024-06-25 17:00:23.669246] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0A8-0-0] [lt=9] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=303, this=0x2b3a9537b8d0, extra_info="palf_id:1006") [2024-06-25 17:00:23.669249] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0A8-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=26, avg_log_submit_to_flush_time=86, avg_log_submit_to_slide_time=277) [2024-06-25 17:00:23.670765] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=10064058, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=162, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:23.672377] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.679620] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62232][pnio1][T1001][YB420ABF4533-00061661718F655F-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.687750] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.690744] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.693581] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.697373] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.703182] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.703797] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF_DUMP](palf_id=1010, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1010, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:96422474, max_lsn:75761243309, max_scn:{val:1719306023291549003, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:75761243309}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:75719300269}}, last_submit_lsn:{lsn:75761243187}, last_submit_end_lsn:{lsn:75761243309}, last_submit_log_id:96422474, last_submit_log_pid:1, max_flushed_lsn:{lsn:75761243187}, max_flushed_end_lsn:{lsn:75761243309}, max_flushed_log_pid:1, committed_end_lsn:{lsn:75761243309}, last_slide_log_id:96422474, last_slide_scn:{val:1719306023291549003, v:0}, last_slide_lsn:{lsn:75761243187}, last_slide_end_lsn:{lsn:75761243309}, last_slide_log_pid:1, last_slide_log_accum_checksum:2787930530, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2b3aaf16e530}, [StateMgr]={this:0x2b3aaf1722b0, self:"10.191.69.51:2882", palf_id:1010, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, leader:"10.189.192.96:2882", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:96422475, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x2b3aaf178b70, allow_vote:true}, [ConfigMgr]={palf_id:1010, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:1, config_seq:2}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1719306021902771, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1719306021902771, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2b3aaf16fc70}, [ModeMgr]={palf_id:1010, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697231063001, v:0}}, accepted_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697231063001, v:0}}, last_submit_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697231063001, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1010, is_inited:true, min_block_max_scn:{val:1718632579709288001, v:0}, min_block_id:1116, min_block_min_scn:{val:1718582295147241001, v:0}, base_lsn_for_block_gc:{lsn:75627073536}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, log_config_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:0, config_seq:1}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:2}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:-1, lock_time:-1}}, prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_mode_pid:1}, log_snapshot_meta:{version:1, base_lsn:{lsn:75627073536}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713323697231063001, v:0}}}, log_meta_storage:{log_tail:{lsn:3055616}, readable_log_tail:{lsn:3055616}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:75761283072}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1010}, block_mgr:{log_dir:"/data/clog/tenant_1002/1010/meta", dir_fd:827, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:64049152, block_header_serialize_buf_:0x2b3aaf174438, flashback_version:0}, log_storage:{log_tail:{lsn:75761243309}, readable_log_tail:{lsn:75761243309}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:75694178304}, min_scn:{val:1719249456795884001, v:0}, curr_block_id:1128, palf_id:1010}, block_mgr:{log_dir:"/data/clog/tenant_1002/1010/log", dir_fd:828, min_block_id:1116, max_block_id:1129, curr_writable_block_id:1128}, logical_block_size_:67104768, curr_block_writable_size_:39763, block_header_serialize_buf_:0x2b3aaf1760b8, flashback_version:0}, palf_epoch:5, last_purge_throttling_ts:-1, this:0x2b3aaf1729f0}, [Reconfirm]={palf_id:1010, self:"10.191.69.51:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2b3aaf172430}) [2024-06-25 17:00:23.703908] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=109][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1010, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:23.703914] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF_DUMP](palf_id=1010, self="10.191.69.51:2882", ack_info_list=[]) [2024-06-25 17:00:23.703918] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1010, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:23.703922] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=3] current log_task status(palf_id=1010, self="10.191.69.51:2882", log_id=96422475, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[]) [2024-06-25 17:00:23.705154] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100136][T1004_L0_G0][T1004][YB420ABF4539-000616463E4CF4F3-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:23.708560] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.709946] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100127][T1004_L0_G0][T1004][YB420ABF4539-000616463E4CF4F3-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:23.711398] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.712140] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=3] [PALF_DUMP](palf_id=1039, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1039, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:23447728, max_lsn:2854441918, max_scn:{val:1719306023465151001, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:2854441918}, reserved_buffer_size:41943040, available_buffer_size:33554432, readable_begin_lsn:{lsn:2812498878}}, last_submit_lsn:{lsn:2854441796}, last_submit_end_lsn:{lsn:2854441918}, last_submit_log_id:23447728, last_submit_log_pid:1, max_flushed_lsn:{lsn:2854441796}, max_flushed_end_lsn:{lsn:2854441918}, max_flushed_log_pid:1, committed_end_lsn:{lsn:2854441918}, last_slide_log_id:23447728, last_slide_scn:{val:1719306023465151001, v:0}, last_slide_lsn:{lsn:2854441796}, last_slide_end_lsn:{lsn:2854441918}, last_slide_log_pid:1, last_slide_log_accum_checksum:401508727, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2bcde0524530}, [StateMgr]={this:0x2bcde05282b0, self:"10.191.69.51:2882", palf_id:1039, role:"LEADER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, leader:"10.191.69.51:2882", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:23447729, is_changing_config_with_arb:false, reconfirm_start_time_us:1716928608989851, palf_role_change_cb:0x2bcde052eb70, allow_vote:true}, [ConfigMgr]={palf_id:1039, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.173:2882", timestamp:1716928619820263, flag:0}{server:"10.186.69.177:2882", timestamp:1716928774338690, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:1, config_seq:17}, log_sync_memberlist:6{server:"10.186.69.173:2882", timestamp:1716928619820263, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.186.69.177:2882", timestamp:1716928774338690, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:6, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17235, lock_time:1716928615395690}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:18}, log_sync_memberlist:5{server:"10.186.69.173:2882", timestamp:1716928619820263, flag:0}{server:"10.186.69.177:2882", timestamp:1716928774338690, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17235, lock_time:1716928615395690}}, prev_log_proposal_id:1, prev_lsn:{lsn:535732}, prev_mode_pid:1}, running_args:{server:{server:"10.186.69.176:2882", timestamp:1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:1, config_seq:16}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"REMOVE_MEMBER_AND_NUM"}, state:0, checking_barrier:{prev_log_proposal_id:1, prev_lsn:{lsn:535732}, prev_end_lsn:{lsn:535854}, prev_mode_pid:1}, reconfig_barrier:{prev_log_proposal_id:1, prev_lsn:{lsn:535732}, prev_end_lsn:{lsn:535854}, prev_mode_pid:1}, persistent_config_version:{proposal_id:1, config_seq:18}, ms_ack_list:0, resend_config_version:{proposal_id:1, config_seq:18}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:1716928923463983, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:-1, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:-1, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2bcde0525c70}, [ModeMgr]={palf_id:1039, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1716928602493732000, v:0}}, accepted_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1716928602493732000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1716928602493732000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:1716928609189983, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1039, is_inited:true, min_block_max_scn:{val:1718674103356367004, v:0}, min_block_id:30, min_block_min_scn:{val:1718619343469857005, v:0}, base_lsn_for_block_gc:{lsn:2818400256}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, log_config_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:1, config_seq:17}, log_sync_memberlist:6{server:"10.186.69.173:2882", timestamp:1716928619820263, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.186.69.177:2882", timestamp:1716928774338690, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:6, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17235, lock_time:1716928615395690}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:18}, log_sync_memberlist:5{server:"10.186.69.173:2882", timestamp:1716928619820263, flag:0}{server:"10.186.69.177:2882", timestamp:1716928774338690, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17235, lock_time:1716928615395690}}, prev_log_proposal_id:1, prev_lsn:{lsn:535732}, prev_mode_pid:1}, log_snapshot_meta:{version:1, base_lsn:{lsn:2818400256}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1716928602493732000, v:0}}}, log_meta_storage:{log_tail:{lsn:258048}, readable_log_tail:{lsn:258048}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:2885505024}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1039}, block_mgr:{log_dir:"/data/clog/tenant_1004/1039/meta", dir_fd:883, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:66846720, block_header_serialize_buf_:0x2bcde052a438, flashback_version:0}, log_storage:{log_tail:{lsn:2854441918}, readable_log_tail:{lsn:2854441918}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:2818400256}, min_scn:{val:1719276629131736002, v:0}, curr_block_id:42, palf_id:1039}, block_mgr:{log_dir:"/data/clog/tenant_1004/1039/log", dir_fd:983, min_block_id:30, max_block_id:43, curr_writable_block_id:42}, logical_block_size_:67104768, curr_block_writable_size_:31063106, block_header_serialize_buf_:0x2bcde052c0b8, flashback_version:0}, palf_epoch:35, last_purge_throttling_ts:1716928608989871, this:0x2bcde05289f0}, [Reconfirm]={palf_id:1039, self:"10.191.69.51:2882", state:"FINISHED", new_proposal_id:1, prepare_log_ack_list:3"10.189.192.96:2882";"10.186.69.176:2882";"10.186.69.171:2882";, curr_paxos_follower_list:4{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, majority_cnt:3, majority_max_log_server:"10.189.192.96:2882", majority_max_accept_pid:0, majority_max_lsn:{lsn:0}, saved_end_lsn:{lsn:0}, last_submit_prepare_req_time_us:1716928608990010, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2bcde0528430}) [2024-06-25 17:00:23.712267] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=124] [PALF_DUMP](palf_id=1039, self="10.191.69.51:2882", ack_info_list=[{member:{server:"10.186.69.173:2882", timestamp:1716928619820263, flag:0}, last_ack_time_us:1719306023633285, last_flushed_end_lsn:{lsn:2854441918}}, {member:{server:"10.186.69.177:2882", timestamp:1716928774338690, flag:0}, last_ack_time_us:1719306023632932, last_flushed_end_lsn:{lsn:2854441918}}, {member:{server:"10.189.192.96:2882", timestamp:1, flag:0}, last_ack_time_us:1719306023632308, last_flushed_end_lsn:{lsn:2854441918}}, {member:{server:"10.191.69.51:2882", timestamp:1, flag:0}, last_ack_time_us:1719306023632157, last_flushed_end_lsn:{lsn:2854441918}}, {member:{server:"10.191.69.56:2882", timestamp:1, flag:0}, last_ack_time_us:1719306023632271, last_flushed_end_lsn:{lsn:2854441918}}]) [2024-06-25 17:00:23.712279] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=11] current log_task status(palf_id=1039, self="10.191.69.51:2882", log_id=23447729, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[{member:{server:"10.186.69.173:2882", timestamp:1716928619820263, flag:0}, last_ack_time_us:1719306023633285, last_flushed_end_lsn:{lsn:2854441918}}, {member:{server:"10.186.69.177:2882", timestamp:1716928774338690, flag:0}, last_ack_time_us:1719306023632932, last_flushed_end_lsn:{lsn:2854441918}}, {member:{server:"10.189.192.96:2882", timestamp:1, flag:0}, last_ack_time_us:1719306023632308, last_flushed_end_lsn:{lsn:2854441918}}, {member:{server:"10.191.69.51:2882", timestamp:1, flag:0}, last_ack_time_us:1719306023632157, last_flushed_end_lsn:{lsn:2854441918}}, {member:{server:"10.191.69.56:2882", timestamp:1, flag:0}, last_ack_time_us:1719306023632271, last_flushed_end_lsn:{lsn:2854441918}}]) [2024-06-25 17:00:23.715225] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.721074] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.721312] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=8][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.722728] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=5][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.722747] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:103) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=0] already timeout, do not need sleep(sleep_us=0, remain_us=29946140, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=0, timeout_timestamp=1719306053668887) [2024-06-25 17:00:23.724040] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99933][T1004_L0_G0][T1004][YB420ABA45AB-000616C7B3DFED25-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:23.727715] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=25000, remain_us=28303720, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=25, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.728368] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100008][T1004_L0_G0][T1004][YB420ABA45AB-000616C7B3DFED25-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:23.728577] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164323C5053A-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5386, fly_ts=22301565, send_timestamp=1719306001427011) [2024-06-25 17:00:23.728720] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:23.729098] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.730898] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.732133] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=21, this=0x2bcde0525470, extra_info="") [2024-06-25 17:00:23.732147] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b589fc96e50, extra_info="id:1039") [2024-06-25 17:00:23.732152] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:23.732180] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:23.732184] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.732187] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.732190] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.732216] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=67, this=0x2bcde052f3b0, extra_info="palf_id:1039") [2024-06-25 17:00:23.732229] INFO [LIB] stat (utility.h:1214) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbff3b0, extra_info="") [2024-06-25 17:00:23.732238] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2bcde052f5e0, extra_info="palf_id:1039") [2024-06-25 17:00:23.732374] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164325962BAA-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301561, send_timestamp=1719306001430812) [2024-06-25 17:00:23.732417] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BAA-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2bcde05256a0, extra_info="palf_id:1039") [2024-06-25 17:00:23.732425] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BAA-0-0] [lt=7] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=300, this=0x2bcde05258d0, extra_info="palf_id:1039") [2024-06-25 17:00:23.734832] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:23.739262] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=0) [2024-06-25 17:00:23.740573] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.748138] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=12][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:23.748147] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [99177][T1004_TenantInf][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:23.748750] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.751785] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=18073193, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=110, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:23.761962] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [62160][ServerGTimer][T1004][Y0-0000000000000000-0-0] [lt=8] get_disk_usage(ret=0, capacity(MB):=2073600, used(MB):=1658818) [2024-06-25 17:00:23.762449] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164325C3018B-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301561, send_timestamp=1719306001460887) [2024-06-25 17:00:23.762475] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C3018B-0-0] [lt=7] [PALF STAT GROUP LOG INFO](palf_id=1010, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:23.762752] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABF4533-00061661728FC991-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301561, send_timestamp=1719306001461190) [2024-06-25 17:00:23.762787] INFO [SQL.PC] update_memory_conf (ob_plan_cache.cpp:1916) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=3] update plan cache memory config(ob_plan_cache_percentage=5, ob_plan_cache_evict_high_percentage=90, ob_plan_cache_evict_low_percentage=50, tenant_id=1003) [2024-06-25 17:00:23.762797] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1457) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=9] start lib cache evict(tenant_id=1003, mem_hold=230686720, mem_limit=4026531840, cache_obj_num=717, cache_node_num=717) [2024-06-25 17:00:23.762805] INFO [SQL.PC] cache_evict (ob_plan_cache.cpp:1474) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=7] end lib cache evict(tenant_id=1003, cache_evict_num=0, mem_hold=230686720, mem_limit=4026531840, cache_obj_num=717, cache_node_num=717) [2024-06-25 17:00:23.762811] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2668) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=5] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:23.762945] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.765002] INFO [SQL.PC] dump_all_objs (ob_plan_cache.cpp:2387) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=2] Dumping All Cache Objs(alloc_obj_list.count()=717, alloc_obj_list=[{obj_id:14, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176366}, {obj_id:16, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:449104}, {obj_id:17, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303715}, {obj_id:18, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:248080}, {obj_id:19, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:271604}, {obj_id:20, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216033}, {obj_id:21, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191870}, {obj_id:22, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176736}, {obj_id:23, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179794}, {obj_id:24, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230228}, {obj_id:25, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188271}, {obj_id:26, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196930}, {obj_id:28, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174576}, {obj_id:29, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:301933}, {obj_id:33, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182834}, {obj_id:34, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178768}, {obj_id:35, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:211414}, {obj_id:36, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224486}, {obj_id:41, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188622}, {obj_id:42, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199959}, {obj_id:43, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199095}, {obj_id:44, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199093}, {obj_id:45, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193076}, {obj_id:46, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214118}, {obj_id:47, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199546}, {obj_id:48, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199766}, {obj_id:49, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190029}, {obj_id:50, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186863}, {obj_id:51, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185594}, {obj_id:52, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200193}, {obj_id:53, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193157}, {obj_id:54, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188399}, {obj_id:55, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185761}, {obj_id:56, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:57, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185793}, {obj_id:58, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188747}, {obj_id:59, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185738}, {obj_id:60, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192013}, {obj_id:61, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187261}, {obj_id:62, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183532}, {obj_id:63, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210345}, {obj_id:64, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183340}, {obj_id:65, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188170}, {obj_id:66, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183730}, {obj_id:67, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191888}, {obj_id:68, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183436}, {obj_id:69, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184586}, {obj_id:70, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188962}, {obj_id:71, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:308297}, {obj_id:72, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:251495}, {obj_id:73, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:304658}, {obj_id:74, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:247858}, {obj_id:75, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201160}, {obj_id:77, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189501}, {obj_id:81, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176269}, {obj_id:83, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:84, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174856}, {obj_id:85, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:86, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178656}, {obj_id:87, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:89, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:90, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217681}, {obj_id:91, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:92, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:94, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183659}, {obj_id:97, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:100, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:103, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:106, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196176}, {obj_id:107, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188336}, {obj_id:108, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:275361}, {obj_id:109, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179806}, {obj_id:110, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191447}, {obj_id:111, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:275362}, {obj_id:112, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218570}, {obj_id:116, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:118, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:684299}, {obj_id:119, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:120, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190714}, {obj_id:121, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223442}, {obj_id:136, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177206}, {obj_id:137, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188473}, {obj_id:141, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238685}, {obj_id:148, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178773}, {obj_id:149, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174899}, {obj_id:151, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174973}, {obj_id:152, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174974}, {obj_id:153, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174899}, {obj_id:154, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176141}, {obj_id:155, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179296}, {obj_id:156, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178110}, {obj_id:158, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:159, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:160, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175011}, {obj_id:161, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175011}, {obj_id:162, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175061}, {obj_id:163, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175061}, {obj_id:164, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:165, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:167, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:168, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181212}, {obj_id:169, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177355}, {obj_id:170, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178587}, {obj_id:172, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177556}, {obj_id:173, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178796}, {obj_id:174, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178746}, {obj_id:175, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176287}, {obj_id:176, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176601}, {obj_id:177, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179968}, {obj_id:178, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177809}, {obj_id:179, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178207}, {obj_id:180, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179455}, {obj_id:181, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176359}, {obj_id:182, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177615}, {obj_id:183, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178110}, {obj_id:184, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176293}, {obj_id:185, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182472}, {obj_id:186, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181294}, {obj_id:187, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181212}, {obj_id:188, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178855}, {obj_id:189, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179247}, {obj_id:190, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180179}, {obj_id:191, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174869}, {obj_id:192, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176141}, {obj_id:193, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178955}, {obj_id:194, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179556}, {obj_id:195, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176751}, {obj_id:196, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182472}, {obj_id:197, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176028}, {obj_id:198, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176242}, {obj_id:199, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177698}, {obj_id:200, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177506}, {obj_id:201, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176601}, {obj_id:202, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:178207}, {obj_id:203, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176359}, {obj_id:204, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179988}, {obj_id:205, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179191}, {obj_id:206, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:383555}, {obj_id:275, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:291, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176429}, {obj_id:301, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195808}, {obj_id:349, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:435, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:284362}, {obj_id:438, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:277099}, {obj_id:441, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:365724}, {obj_id:442, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:352792}, {obj_id:477, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294946}, {obj_id:490, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210456}, {obj_id:501, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:326767}, {obj_id:503, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230400}, {obj_id:504, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185730}, {obj_id:506, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:12123}, {obj_id:507, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1069986}, {obj_id:508, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:173197}, {obj_id:509, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199042}, {obj_id:510, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201819}, {obj_id:511, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197302}, {obj_id:513, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231048}, {obj_id:514, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222216}, {obj_id:515, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:516, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:519, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:520, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:340122}, {obj_id:635, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:281722}, {obj_id:636, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:313618}, {obj_id:637, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:404380}, {obj_id:638, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:513636}, {obj_id:642, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:406569}, {obj_id:643, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:526721}, {obj_id:644, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:324772}, {obj_id:645, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:385343}, {obj_id:652, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:336141}, {obj_id:653, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:407607}, {obj_id:656, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:302133}, {obj_id:657, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:349539}, {obj_id:718, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234380}, {obj_id:1036, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286678}, {obj_id:1093, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:316722}, {obj_id:1119, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:306729}, {obj_id:1131, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:226842}, {obj_id:1134, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1135, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204208}, {obj_id:1137, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1170, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1172, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1180, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1181, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1184, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:360170}, {obj_id:1205, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:226578}, {obj_id:1209, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234762}, {obj_id:1210, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:175923}, {obj_id:1217, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185026}, {obj_id:1219, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186335}, {obj_id:1223, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186868}, {obj_id:1224, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187165}, {obj_id:1230, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196301}, {obj_id:1231, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:333432}, {obj_id:1242, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1250, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184636}, {obj_id:1277, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219147}, {obj_id:1280, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:183351}, {obj_id:1281, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181880}, {obj_id:1282, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180188}, {obj_id:1283, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:343446}, {obj_id:1349, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212498}, {obj_id:1359, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200164}, {obj_id:1368, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:409123}, {obj_id:1371, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:527267}, {obj_id:1394, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212873}, {obj_id:1396, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194252}, {obj_id:1399, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194540}, {obj_id:1420, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:263353}, {obj_id:1421, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276973}, {obj_id:1428, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:349054}, {obj_id:1429, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:411748}, {obj_id:1432, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:439302}, {obj_id:1436, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276882}, {obj_id:1438, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:298234}, {obj_id:1439, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:553609}, {obj_id:1441, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:425770}, {obj_id:1443, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:532331}, {obj_id:1444, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:441552}, {obj_id:1445, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:557166}, {obj_id:1446, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:235923}, {obj_id:1447, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234251}, {obj_id:1449, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229611}, {obj_id:1471, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:249500}, {obj_id:1472, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:303921}, {obj_id:1473, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:258588}, {obj_id:1475, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320137}, {obj_id:1480, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:282443}, {obj_id:1481, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:362392}, {obj_id:1488, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286101}, {obj_id:1489, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:368900}, {obj_id:1493, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:274640}, {obj_id:1494, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:294693}, {obj_id:1497, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:378392}, {obj_id:1498, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:457842}, {obj_id:1499, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:437052}, {obj_id:1501, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:428020}, {obj_id:1502, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:550068}, {obj_id:1503, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:412238}, {obj_id:1504, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:362578}, {obj_id:1505, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:535888}, {obj_id:1506, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:511046}, {obj_id:1507, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:433015}, {obj_id:1508, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:243979}, {obj_id:1509, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:234595}, {obj_id:1510, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:232619}, {obj_id:1511, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:224335}, {obj_id:1512, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231595}, {obj_id:1513, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223657}, {obj_id:1514, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:346779}, {obj_id:1515, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231905}, {obj_id:1518, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:318041}, {obj_id:1519, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:310697}, {obj_id:1520, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:425065}, {obj_id:1521, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:367809}, {obj_id:1522, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:412013}, {obj_id:1523, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:509887}, {obj_id:1524, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:268389}, {obj_id:1527, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:336988}, {obj_id:1533, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:347500}, {obj_id:1535, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:353092}, {obj_id:1537, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:351196}, {obj_id:1540, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:477250}, {obj_id:1541, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:487084}, {obj_id:1542, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:483796}, {obj_id:1549, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203372}, {obj_id:1555, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:312593}, {obj_id:1556, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:415310}, {obj_id:1559, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207017}, {obj_id:1561, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208699}, {obj_id:1562, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205264}, {obj_id:1564, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216265}, {obj_id:1565, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:215597}, {obj_id:1566, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216631}, {obj_id:1567, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:217307}, {obj_id:1569, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205718}, {obj_id:1571, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:210530}, {obj_id:1577, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202860}, {obj_id:1578, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221503}, {obj_id:1579, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:219106}, {obj_id:1582, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212516}, {obj_id:1586, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184380}, {obj_id:1588, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221044}, {obj_id:1590, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:221684}, {obj_id:1591, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320059}, {obj_id:1599, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180931}, {obj_id:1602, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174016}, {obj_id:1619, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213180}, {obj_id:1642, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189353}, {obj_id:1643, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189220}, {obj_id:1644, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193462}, {obj_id:1645, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180107}, {obj_id:1654, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181394}, {obj_id:1657, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181674}, {obj_id:1658, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189842}, {obj_id:1666, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201840}, {obj_id:1668, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181740}, {obj_id:1669, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190036}, {obj_id:1670, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:182184}, {obj_id:1708, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:200183}, {obj_id:1709, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218850}, {obj_id:1717, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:319852}, {obj_id:1720, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:431890}, {obj_id:1726, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:563003}, {obj_id:1731, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:373531}, {obj_id:1746, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:296698}, {obj_id:1759, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:411595}, {obj_id:1760, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:521508}, {obj_id:1764, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:260624}, {obj_id:1765, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:264190}, {obj_id:1766, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176744}, {obj_id:1767, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174432}, {obj_id:1768, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177085}, {obj_id:1769, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:181110}, {obj_id:1770, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:179110}, {obj_id:1771, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176843}, {obj_id:1773, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:198614}, {obj_id:1776, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:242568}, {obj_id:1777, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245745}, {obj_id:1778, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:249200}, {obj_id:1779, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:514775}, {obj_id:1780, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:174650}, {obj_id:1781, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:256438}, {obj_id:1782, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:176308}, {obj_id:1783, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:269328}, {obj_id:1784, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241927}, {obj_id:1785, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:195211}, {obj_id:1786, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1787, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1789, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:313384}, {obj_id:1790, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189174}, {obj_id:1794, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:201926}, {obj_id:1795, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:263611}, {obj_id:1796, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:330941}, {obj_id:1800, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:252092}, {obj_id:1801, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:1728}, {obj_id:1807, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214817}, {obj_id:1811, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:223289}, {obj_id:1823, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:320604}, {obj_id:1826, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:268234}, {obj_id:1827, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:369950}, {obj_id:1828, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:286091}, {obj_id:1841, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297804}, {obj_id:1842, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:334896}, {obj_id:1848, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:212552}, {obj_id:1850, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204050}, {obj_id:1858, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:239469}, {obj_id:1859, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213099}, {obj_id:1860, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:239048}, {obj_id:1868, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229019}, {obj_id:1869, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237507}, {obj_id:1871, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:230088}, {obj_id:1872, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:238192}, {obj_id:1881, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222260}, {obj_id:1882, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:255313}, {obj_id:1885, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203714}, {obj_id:1886, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205084}, {obj_id:1888, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:225301}, {obj_id:1889, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:229642}, {obj_id:1892, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:237762}, {obj_id:1893, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187591}, {obj_id:1898, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204758}, {obj_id:1899, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213900}, {obj_id:1900, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:202713}, {obj_id:1901, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204051}, {obj_id:1903, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204752}, {obj_id:1904, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:213464}, {obj_id:1905, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189807}, {obj_id:1906, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189568}, {obj_id:1908, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177216}, {obj_id:1909, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189999}, {obj_id:1910, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:190072}, {obj_id:1911, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184744}, {obj_id:1927, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:306481}, {obj_id:1928, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:404932}, {obj_id:1934, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:218446}, {obj_id:1935, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:248651}, {obj_id:1939, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:245943}, {obj_id:1940, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:297508}, {obj_id:1941, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207520}, {obj_id:1942, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206244}, {obj_id:1943, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208189}, {obj_id:1944, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:205856}, {obj_id:1945, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:209158}, {obj_id:1946, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206074}, {obj_id:1947, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:214746}, {obj_id:1948, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206759}, {obj_id:1953, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184677}, {obj_id:1961, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185383}, {obj_id:1962, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:338806}, {obj_id:1963, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:329717}, {obj_id:1964, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:400818}, {obj_id:1965, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:384947}, {obj_id:1966, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:215423}, {obj_id:1967, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193646}, {obj_id:1968, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:180032}, {obj_id:1970, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207111}, {obj_id:1971, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207428}, {obj_id:1972, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:216076}, {obj_id:1973, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:207780}, {obj_id:1982, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191845}, {obj_id:1985, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177576}, {obj_id:1990, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:177576}, {obj_id:1994, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:186606}, {obj_id:1995, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:233374}, {obj_id:1996, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:208113}, {obj_id:1998, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:231846}, {obj_id:1999, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:227382}, {obj_id:2000, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189106}, {obj_id:2001, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:197404}, {obj_id:2002, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189506}, {obj_id:2003, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187054}, {obj_id:2004, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188768}, {obj_id:2005, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:241398}, {obj_id:2006, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:188592}, {obj_id:2007, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:185206}, {obj_id:2008, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189703}, {obj_id:2009, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199149}, {obj_id:2010, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276526}, {obj_id:2012, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:276526}, {obj_id:2013, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:187377}, {obj_id:2015, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:189177}, {obj_id:2016, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191033}, {obj_id:2017, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:192817}, {obj_id:2018, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203855}, {obj_id:2019, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196646}, {obj_id:2020, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:199942}, {obj_id:2021, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203138}, {obj_id:2023, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:193422}, {obj_id:2024, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:222722}, {obj_id:2025, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194506}, {obj_id:2026, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:184181}, {obj_id:2027, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:191378}, {obj_id:2028, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206684}, {obj_id:2029, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:203947}, {obj_id:2030, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206804}, {obj_id:2031, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:204075}, {obj_id:2034, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:194701}, {obj_id:2036, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:206437}, {obj_id:2038, tenant_id:1003, log_del_time:9223372036854775807, real_del_time:9223372036854775807, ref_count:2, added_to_lc:true, mem_used:196470}, {obj_id:2039, tenant_id:1003, log_del_time:922337 [2024-06-25 17:00:23.765475] INFO [SQL.PC] runTimerTask (ob_plan_cache.cpp:2676) [100366][T1003_PlanCache][T1003][Y0-0000000000000000-0-0] [lt=471] schedule next cache evict task(evict_interval=5000000) [2024-06-25 17:00:23.768991] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.769000] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.769003] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.769007] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=4] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:23.769009] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:23.769284] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=2] [APPLY STAT CB APPEND COST TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=32, this=0x2b3a9538a280, extra_info="") [2024-06-25 17:00:23.769294] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=9] [APPLY STAT CB IN QUEUE TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=274, this=0x2b3a9538a4b0, extra_info="") [2024-06-25 17:00:23.769297] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=3] [APPLY STAT CB WAIT COMMIT TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=0, this=0x2b3a9538a6e0, extra_info="") [2024-06-25 17:00:23.769301] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=4] [APPLY STAT CB EXECUTE TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=-512, this=0x2b3a9538a910, extra_info="") [2024-06-25 17:00:23.769304] INFO [LIB] stat (utility.h:1197) [63265][T1002_ApplySrv0][T1002][YB420ABF4533-0006166041EEFF36-0-0] [lt=3] [APPLY STAT CB TOTAL TIME](cur_stat_count=47, stat_interval=5000000, avg (count/cost)=-205, this=0x2b3a9538ab40, extra_info="") [2024-06-25 17:00:23.772284] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [99194][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=6] ====== tenant freeze timer task ====== [2024-06-25 17:00:23.774984] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=5174283, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=194, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:23.776179] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=869162, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=222, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:23.776602] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=0] will sleep(sleep_us=1000, remain_us=29892285, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=1, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:23.777811] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=91000, remain_us=20949365, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=91, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:23.779681] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62234][pnio1][T1001][YB420ABF4533-00061661718F6560-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.782281] INFO [SHARE.PT] get_ls_info_ (ob_rpc_ls_table.cpp:142) [62315][AutoLSLoc][T0][YB420ABF4533-000616613ECF0EF2-0-0] [lt=8] fetch root ls success(local_cluster_id=1713323324, ls_info={tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:0, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.171:2882", sql_port:2881, role:1, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:3, zone:"zone3", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607828518, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.186.69.176:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:4, zone:"zone4", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607829360, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.189.192.96:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:5, zone:"zone5", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607830236, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.51:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, {modify_time_us:1713323607831113, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"10.191.69.56:2882", sql_port:2881, role:2, member_list:[{server:"10.186.69.171:2882", timestamp:1}, {server:"10.186.69.176:2882", timestamp:1}, {server:"10.189.192.96:2882", timestamp:1}, {server:"10.191.69.51:2882", timestamp:1}, {server:"10.191.69.56:2882", timestamp:1}], replica_type:0, proposal_id:0, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:2, zone:"zone2", paxos_replica_number:5, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, rs_list=["10.186.69.171:2882", "10.191.69.51:2882", "10.191.69.56:2882", "10.186.69.176:2882", "10.189.192.96:2882"]) [2024-06-25 17:00:23.782658] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.784380] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62315][AutoLSLoc][T1001][YB420ABF4533-000616613ECF0EF2-0-0] [lt=48][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.787810] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.795775] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62821][PxTargetMgr0][T0][Y0-0000000000000000-0-0] [lt=8][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.802416] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.806304] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=1] will sleep(sleep_us=26000, remain_us=28225131, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=26, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.806651] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC0C0-0-0] [lt=9] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a5d1db470, extra_info="") [2024-06-25 17:00:23.806752] INFO [LIB] stat (utility.h:1197) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=11] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=80, this=0x2b3a5d1e53b0, extra_info="palf_id:1") [2024-06-25 17:00:23.806765] INFO [LIB] stat (utility.h:1214) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=10] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=6, this=0x2b3a40ffddb0, extra_info="") [2024-06-25 17:00:23.806781] INFO [LIB] stat (utility.h:1197) [62591][T1_LogIOCb0][T1][Y0-0000000000000000-0-0] [lt=10] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=10, this=0x2b3a5d1e55e0, extra_info="palf_id:1") [2024-06-25 17:00:23.812967] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62746][T1_ReplaySrv3][T1][Y0-0000000000000000-0-0] [lt=14] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:23.813355] WDIAG [RPC] send (ob_poc_rpc_proxy.h:156) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=0][errcode=-4012] sync rpc execute fail(ret=-4012, addr="10.191.69.52:2882", pcode=1308) [2024-06-25 17:00:23.813368] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=9][errcode=-4012] rpc task_execute fail(ret=-4012, tenant_id=1002, svr="10.191.69.52:2882", timeout=1999716, timeout_timestamp=1719306023812256) [2024-06-25 17:00:23.813377] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=8][errcode=-4012] task execute failed(ret=-4012) [2024-06-25 17:00:23.813382] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=4][errcode=-4012] execute with sql failed(ret=-4012) [2024-06-25 17:00:23.813387] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=4][errcode=-4012] fail execute plan(ret=-4012) [2024-06-25 17:00:23.813391] WDIAG [SQL] open (ob_result_set.cpp:157) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=3][errcode=-4012] execute plan failed(ret=-4012) [2024-06-25 17:00:23.813394] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=2][errcode=-4012] open result set failed(ret=-4012) [2024-06-25 17:00:23.813398] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=3][errcode=-4012] result set open failed(ret=-4012, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}) [2024-06-25 17:00:23.813404] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=5][errcode=-4012] execute failed(ret=-4012, tenant_id=1002, executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1718209168713032) [2024-06-25 17:00:23.813410] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=6][errcode=-4012] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4012, err_:"OB_TIMEOUT", retry_type:0, client_ret:-4012}, need_retry=false) [2024-06-25 17:00:23.813422] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=9][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:23.813425] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=3][errcode=-4012] result set close failed(ret=-4012) [2024-06-25 17:00:23.813428] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=3][errcode=-4012] failed to close result(close_ret=-4012, ret=-4012) [2024-06-25 17:00:23.813434] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=3][errcode=-4012] failed to process record(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, record_ret=-4012, ret=-4012) [2024-06-25 17:00:23.813440] INFO [SERVER] process_final (ob_inner_sql_connection.cpp:734) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=5] slow inner sql(last_ret=-4012, sql={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, process_time=2000938) [2024-06-25 17:00:23.813444] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63252][T1002_FreInfoRe][T1002][YB420ABF4533-000616616D9E8644-0-0] [lt=4][errcode=-4012] failed to process final(executor={ObIExecutor:, sql:"SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name"}, aret=-4012, ret=-4012) [2024-06-25 17:00:23.813450] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] execute sql failed(ret=-4012, tenant_id=1002, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:23.813454] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] retry_while_no_tenant_resource failed(ret=-4012, tenant_id=1002) [2024-06-25 17:00:23.813459] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] execute_read failed(ret=-4012, cluster_id=1713323324, tenant_id=1002) [2024-06-25 17:00:23.813463] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] query failed(ret=-4012, conn=0x2b3b6150d190, start=1719306021812495, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:23.813469] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4012] read failed(ret=-4012) [2024-06-25 17:00:23.813473] WDIAG [SHARE] load (ob_core_table_proxy.cpp:436) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] execute sql failed(ret=-4012, ret="OB_TIMEOUT", tenant_id=1002, sql=SELECT row_id, column_name, column_value FROM __all_core_table WHERE table_name = '__all_global_stat' ORDER BY row_id, column_name) [2024-06-25 17:00:23.813505] WDIAG [SHARE] load (ob_core_table_proxy.cpp:368) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] load failed(ret=-4012, for_update=false) [2024-06-25 17:00:23.813510] WDIAG [SHARE] get (ob_global_stat_proxy.cpp:442) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4012] core_table load failed(ret=-4012, ret="OB_TIMEOUT") [2024-06-25 17:00:23.813513] WDIAG [SHARE] get_snapshot_gc_scn (ob_global_stat_proxy.cpp:164) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] get failed(ret=-4012) [2024-06-25 17:00:23.813517] WDIAG [STORAGE] get_global_info (ob_tenant_freeze_info_mgr.cpp:811) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] fail to get global info(ret=-4012, tenant_id=1002) [2024-06-25 17:00:23.813522] WDIAG [STORAGE] try_update_info (ob_tenant_freeze_info_mgr.cpp:954) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4012] failed to get global info(ret=-4012) [2024-06-25 17:00:23.813526] WDIAG [STORAGE] runTimerTask (ob_tenant_freeze_info_mgr.cpp:1008) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4012] fail to try update info(tmp_ret=-4012, tmp_ret="OB_TIMEOUT") [2024-06-25 17:00:23.813533] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1044) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=2] success to update min reserved snapshot(reserved_snapshot=1718345476433708000, duration=1800, snapshot_gc_ts_=1718347276433708000) [2024-06-25 17:00:23.813550] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1071) [63252][T1002_FreInfoRe][T1002][Y0-0000000000000000-0-0] [lt=5] update reserved snapshot finished(cost_ts=19, reserved_snapshot=1718345476433708000) [2024-06-25 17:00:23.814314] INFO pktc_resp_cb_on_msg (pktc_resp.h:70) [62233][pnio1][T0][Y0-0000000000000000-0-0] [lt=9] PNIO resp cb not found: packet_id=1035878653 [2024-06-25 17:00:23.817886] INFO [STORAGE.BLKMGR] defragment (ob_shared_macro_block_manager.cpp:445) [62683][T1_SSTableDefra][T1][Y0-0000000000000000-0-0] [lt=3] skip defragment task(ret=0, need_defragment=false) [2024-06-25 17:00:23.817896] INFO [STORAGE.BLKMGR] defragment (ob_shared_macro_block_manager.cpp:485) [62683][T1_SSTableDefra][T1][Y0-0000000000000000-0-0] [lt=9] successfully defragment data blocks(ret=0, rewrite_cnt=0, block_used_size_.count()=6) [2024-06-25 17:00:23.817900] INFO [STORAGE.BLKMGR] defragment (ob_shared_macro_block_manager.cpp:503) [62683][T1_SSTableDefra][T1][Y0-0000000000000000-0-0] [lt=4] successfully defragment data blocks(rewrite_cnt=0) [2024-06-25 17:00:23.821593] INFO do_work (ob_rl_mgr.cpp:709) [62262][rl_mgr0][T0][Y0-0000000000000000-0-0] [lt=9] swc wakeup.(stat_period_=1000000, ready=false) [2024-06-25 17:00:23.824639] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9910185, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=163, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:23.825127] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=9] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289107427328, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:23.827589] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95800][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBB9A-0-0] [lt=2] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1211) [2024-06-25 17:00:23.827871] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC959-0-0] [lt=11] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:23.828627] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.831458] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=0] will sleep(sleep_us=2000, remain_us=29837430, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=2, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:23.832187] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:23.832216] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:23.832220] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.832222] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.832225] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.832284] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1039, self="10.191.69.51:2882", committed size=1211) [2024-06-25 17:00:23.832428] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABDC060-0006164325962BAF-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301556, send_timestamp=1719306001530871) [2024-06-25 17:00:23.832444] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325962BB0-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301562, send_timestamp=1719306001530881) [2024-06-25 17:00:23.832468] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BAF-0-0] [lt=6] [PALF STAT LOG TASK TIME](palf_id=1039, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=1, avg_log_gen_to_submit_time=26, avg_log_submit_to_flush_time=88, avg_log_submit_to_slide_time=275) [2024-06-25 17:00:23.832502] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BB0-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1005, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=18, avg_log_submit_to_flush_time=88, avg_log_submit_to_slide_time=276) [2024-06-25 17:00:23.834885] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:23.840628] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.841246] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABF4533-0006166172BEBEEA-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301560, send_timestamp=1719306001539686) [2024-06-25 17:00:23.843887] INFO [COMMON] process (ob_rpc_processor_simple.cpp:2405) [95801][T1004_L0_G19][T1004][YB420ABDC061-00061643A3887520-0-0] [lt=8] handle ObQueryLSIsValidMember requeset(arg_={tenant_id_:1004, self_addr_:"10.189.192.97:2882", ls_array_:[{id:1039}]}) [2024-06-25 17:00:23.843902] INFO [CLOG] get_member_gc_stat (ob_log_handler.cpp:1509) [95801][T1004_L0_G19][T1004][YB420ABDC061-00061643A3887520-0-0] [lt=11] get_member_gc_stat(is_valid_member=false, stat=1, member_list=5{server:"10.186.69.173:2882", timestamp:1716928619820263, flag:0}{server:"10.186.69.177:2882", timestamp:1716928774338690, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, learner_list={learner_num:0, learner_array:[]}, addr="10.189.192.97:2882", this={role:1, proposal_id:1, palf_env_:0x2b3afcbf8030, is_in_stop_state_:false, is_inited_:true, id_:1039}) [2024-06-25 17:00:23.848503] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.848610] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [62162][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=4] trace max work area(auto_calc=true, tenant_max_memory_limit=1073741824, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=53687091, max_wa_memory_size=53687091, tmp_max_wa_memory_size=53687091, pre_mem_target=53687091, remain_memory_size=53687091, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=343166976, washable_size=-2, max_workarea_memory_size=53687091, max_tenant_memory_size=730574848, wash_ratio=6, tenant_id=1001) [2024-06-25 17:00:23.848657] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [62162][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=17] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=6710886, calc_global_bound_size=1048576000000, mem_target_=53687091, wa_max_memory_size=53687091, profile_cnt_=0, total_memory_size=0, max_wa_size=53687091, avg_bound_size=6710886, max_bound_size=6710886, min_bound_size_=1048576) [2024-06-25 17:00:23.848666] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [62162][SqlMemTimer][T1001][Y0-0000000000000000-0-0] [lt=8] timer to calc global bound size(ret=0, global_bound_size_=6710886, manual_calc_cnt_=0, drift_size_=0, pre_drift_size=0, wa_max_memory_size=53687091, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1001, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=6710886, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=53687091, auto_calc=true, sql_mem_callback_.get_total_dump_size()=0) [2024-06-25 17:00:23.848692] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [62162][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=8] trace max work area(auto_calc=true, tenant_max_memory_limit=1073741824, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=53687091, max_wa_memory_size=53687091, tmp_max_wa_memory_size=53687091, pre_mem_target=53687091, remain_memory_size=53687091, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=933953536, washable_size=-2, max_workarea_memory_size=53687091, max_tenant_memory_size=139788288, wash_ratio=6, tenant_id=1002) [2024-06-25 17:00:23.848723] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [62162][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=7] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=6710886, calc_global_bound_size=1048576000000, mem_target_=53687091, wa_max_memory_size=53687091, profile_cnt_=0, total_memory_size=0, max_wa_size=53687091, avg_bound_size=6710886, max_bound_size=6710886, min_bound_size_=1048576) [2024-06-25 17:00:23.848729] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [62162][SqlMemTimer][T1002][Y0-0000000000000000-0-0] [lt=6] timer to calc global bound size(ret=0, global_bound_size_=6710886, manual_calc_cnt_=19494, drift_size_=0, pre_drift_size=0, wa_max_memory_size=53687091, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1002, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=6710886, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=53687091, auto_calc=true, sql_mem_callback_.get_total_dump_size()=11170713168) [2024-06-25 17:00:23.848743] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [62162][SqlMemTimer][T1003][Y0-0000000000000000-0-0] [lt=5] trace max work area(auto_calc=true, tenant_max_memory_limit=80530636800, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=4026531840, max_wa_memory_size=4026531840, tmp_max_wa_memory_size=4026531840, pre_mem_target=4026531840, remain_memory_size=4026531840, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=845557760, washable_size=-2, max_workarea_memory_size=4026531840, max_tenant_memory_size=79685079040, wash_ratio=6, tenant_id=1003) [2024-06-25 17:00:23.848772] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [62162][SqlMemTimer][T1003][Y0-0000000000000000-0-0] [lt=7] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=503316480, calc_global_bound_size=1048576000000, mem_target_=4026531840, wa_max_memory_size=4026531840, profile_cnt_=0, total_memory_size=0, max_wa_size=4026531840, avg_bound_size=503316480, max_bound_size=503316480, min_bound_size_=1048576) [2024-06-25 17:00:23.848777] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [62162][SqlMemTimer][T1003][Y0-0000000000000000-0-0] [lt=5] timer to calc global bound size(ret=0, global_bound_size_=503316480, manual_calc_cnt_=0, drift_size_=0, pre_drift_size=0, wa_max_memory_size=4026531840, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1003, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=503316480, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=4026531840, auto_calc=true, sql_mem_callback_.get_total_dump_size()=0) [2024-06-25 17:00:23.848796] INFO [SQL.ENG] get_max_work_area_size (ob_tenant_sql_memory_manager.cpp:954) [62162][SqlMemTimer][T1004][Y0-0000000000000000-0-0] [lt=4] trace max work area(auto_calc=true, tenant_max_memory_limit=724775731200, total_alloc_size=0, tenant_work_area_memory_hold=0, tenant_work_area_max_size=36238786560, max_wa_memory_size=36238786560, tmp_max_wa_memory_size=36238786560, pre_mem_target=36238786560, remain_memory_size=36238786560, ratio=-nan, alloc_ratio=0.000000000000000000e+00, hold_ratio=0.000000000000000000e+00, tenant_memory_hold=622220120064, washable_size=-2, max_workarea_memory_size=36238786560, max_tenant_memory_size=102555611136, wash_ratio=6, tenant_id=1004) [2024-06-25 17:00:23.848827] INFO [SQL.ENG] calculate_global_bound_size (ob_tenant_sql_memory_manager.cpp:323) [62162][SqlMemTimer][T1004][Y0-0000000000000000-0-0] [lt=7] timer to calc global bound size(ret=0, best_interval_idx=1099, global_bound_size_=4529848320, calc_global_bound_size=1048576000000, mem_target_=36238786560, wa_max_memory_size=36238786560, profile_cnt_=0, total_memory_size=0, max_wa_size=36238786560, avg_bound_size=4529848320, max_bound_size=4529848320, min_bound_size_=1048576) [2024-06-25 17:00:23.848832] INFO [SQL.ENG] calculate_global_bound_size_by_interval_info (ob_tenant_sql_memory_manager.cpp:1202) [62162][SqlMemTimer][T1004][Y0-0000000000000000-0-0] [lt=5] timer to calc global bound size(ret=0, global_bound_size_=4529848320, manual_calc_cnt_=6994, drift_size_=0, pre_drift_size=0, wa_max_memory_size=36238786560, sql_mem_callback_.get_total_alloc_size()=0, tenant_id_=1004, profile_cnt_=0, pre_profile_cnt_=0, pre_profile_cnt=0, calc_info.get_global_bound_size()=4529848320, total_memory_size=0, cur_profile_cnt=0, calc_info.get_mem_target()=36238786560, auto_calc=true, sql_mem_callback_.get_total_dump_size()=542226249464) [2024-06-25 17:00:23.862496] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABDC060-0006164325C3018F-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301557, send_timestamp=1719306001560938) [2024-06-25 17:00:23.862798] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABF4533-00061661728FC994-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301566, send_timestamp=1719306001561232) [2024-06-25 17:00:23.862819] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABF4533-00061661728FC994-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1010, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=95, avg_log_submit_to_slide_time=289) [2024-06-25 17:00:23.862824] INFO [PALF] read (log_cache.cpp:100) [63317][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT HOT CACHE HIT RATE](palf_id=1010, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:23.864692] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1BE1-0-0] [lt=9] [PALF STAT COMMITTED LOG SIZE](palf_id=1009, self="10.191.69.51:2882", committed size=1098) [2024-06-25 17:00:23.864771] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1098) [2024-06-25 17:00:23.869047] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.869055] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.869058] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.869061] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:23.869064] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:23.878687] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-0006164324713065-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=22301554, send_timestamp=1719306001577133) [2024-06-25 17:00:23.879691] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7945-0-0] [lt=5] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12239, add_timestamp:1719306023879684}) [2024-06-25 17:00:23.879716] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [62786][BlackListServic][T1][YB420ABF4533-000616616F5E7945-0-0] [lt=10] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12287, add_timestamp:1719306023879715}) [2024-06-25 17:00:23.879714] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF06-0-0] [lt=27] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12239, locations=[{table_id:12239, partition_id:12239, partition_cnt:0, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023879701, sql_renew_time:1719306023879701, is_mark_fail:false}]) [2024-06-25 17:00:23.879730] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF06-0-0] [lt=16] success to process renew task(task={tenant_id:1, table_id:12239, add_timestamp:1719306023879684}, locations=[{table_id:12239, partition_id:12239, partition_cnt:0, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023879701, sql_renew_time:1719306023879701, is_mark_fail:false}]) [2024-06-25 17:00:23.879742] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF07-0-0] [lt=9] fetch virtual table location with tenant(rpc tenant=0, tenant_id=1, table_id=12287, vtable_type={tenant_id:1, type:4}) [2024-06-25 17:00:23.879862] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62237][pnio1][T1001][YB420ABF4533-00061661718F6561-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.880549] INFO [SHARE.LOCATION] renew_vtable_location_ (ob_vtable_location_service.cpp:207) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF07-0-0] [lt=7] renew vtable location success(ret=0, ret="OB_SUCCESS", tenant_id=1, table_id=12287, locations=[{table_id:12287, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:23.880574] INFO [SHARE.LOCATION] batch_process_tasks (ob_vtable_location_service.cpp:414) [62323][VTblLocAsyncUp0][T0][YB420ABF4533-000616616CFEFF07-0-0] [lt=25] success to process renew task(task={tenant_id:1, table_id:12287, add_timestamp:1719306023879715}, locations=[{table_id:12287, partition_id:0, partition_cnt:10, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:1, partition_cnt:10, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:2, partition_cnt:10, replica_locations:[{server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:3, partition_cnt:10, replica_locations:[{server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:4, partition_cnt:10, replica_locations:[{server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:5, partition_cnt:10, replica_locations:[{server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:6, partition_cnt:10, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:7, partition_cnt:10, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:8, partition_cnt:10, replica_locations:[{server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}, {table_id:12287, partition_id:9, partition_cnt:10, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, reserved:0, property:{memstore_percent_:100}}], renew_time:1719306023880542, sql_renew_time:0, is_mark_fail:false}]) [2024-06-25 17:00:23.881451] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABF4533-0006166172BEBEEF-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301555, send_timestamp=1719306001579895) [2024-06-25 17:00:23.882192] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63280][T1002_LSMetaCh][T1001][YB420ABF4533-000616615C0FE85C-0-0] [lt=1][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.883182] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [62786][BlackListServic][T1001][YB420ABF4533-000616616F5E7945-0-0] [lt=5][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:23.883290] INFO [STORAGE.TRANS] run1 (ob_black_list.cpp:194) [62786][BlackListServic][T0][Y0-0000000000000000-0-0] [lt=0] ls blacklist refresh finish(cost_time=3935) [2024-06-25 17:00:23.885847] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=27000, remain_us=28145589, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=27, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.887281] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=0] will sleep(sleep_us=3000, remain_us=29781606, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=3, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:23.887870] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.893559] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100322][T1004_L0_G0][T1004][YB420ABF4538-00061669369F3015-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:23.897880] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99933][T1004_L0_G0][T1004][YB420ABF4538-00061669369F3015-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:23.902248] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=25] get_max_applied_scn(scn={val:1713323697227896000, v:0}, this={ls_id_:{id:1007}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713323697227896000, v:0}}) [2024-06-25 17:00:23.902262] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=12] get_min_unreplayed_log_info(lsn={lsn:85187192661}, scn={val:1719306023372687005, v:0}, this={ls_id_:{id:1007}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306023850417, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:85187192661}, next_to_submit_scn_:{val:1719306023372687005, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713323697227896000, v:0}, iterator_:{iterator_impl:{buf_:0x2b4185039000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:85187192539}, end_lsn:{lsn:85187192661}, read_buf:{buf_len_:86016, buf_:0x2b4185039000}, block_size:67104768, log_storage_:0x2b3a86d23470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306023372687004, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306023372687004, v:0}, data_checksum:2442502576, flag:0}}, init_mode_version:0, accumulate_checksum:164310715, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306023372687004, v:0}}}}}) [2024-06-25 17:00:23.903906] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=10] re_register_parent reason(palf_id=1008, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306019901686, last_first_register_time_us=1713323715481338, register_time_us=1719306019901686, curr_time_us=1719306023903906) [2024-06-25 17:00:23.903928] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=10] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882", reason=1, leader="10.186.69.171:2882", register_time_us=1719306023903917, last_submit_register_req_time_us=1719306023903917, register_parent_reason=1) [2024-06-25 17:00:23.903934] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=6] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882") [2024-06-25 17:00:23.905496] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=17919483, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=111, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:23.911459] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.912203] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2648) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=17] [PALF_DUMP](palf_id=1008, self="10.191.69.51:2882", [SlidingWindow]={palf_id:1008, self:"10.191.69.51:2882", lsn_allocator:{max_log_id:404884535, max_lsn:5880977363420, max_scn:{val:1719306023643165002, v:0}}, group_buffer:{log_group_buffer: start_lsn:{lsn:0}, reuse_lsn:{lsn:5880977363420}, reserved_buffer_size:41943040, available_buffer_size:41943040, readable_begin_lsn:{lsn:5880935420380}}, last_submit_lsn:{lsn:5880977363298}, last_submit_end_lsn:{lsn:5880977363420}, last_submit_log_id:404884535, last_submit_log_pid:1, max_flushed_lsn:{lsn:5880977363298}, max_flushed_end_lsn:{lsn:5880977363420}, max_flushed_log_pid:1, committed_end_lsn:{lsn:5880977363420}, last_slide_log_id:404884535, last_slide_scn:{val:1719306023643165002, v:0}, last_slide_lsn:{lsn:5880977363298}, last_slide_end_lsn:{lsn:5880977363420}, last_slide_log_pid:1, last_slide_log_accum_checksum:440433908, last_fetch_end_lsn:{lsn:18446744073709551615}, last_fetch_max_log_id:-1, last_fetch_committed_end_lsn:{lsn:18446744073709551615}, last_truncate_lsn:{lsn:18446744073709551615}, last_fetch_req_time:0, is_truncating:false, is_rebuilding:false, last_rebuild_lsn:{lsn:18446744073709551615}, freeze_mode:"FEEDBACK_FREEZE_MODE", has_pending_handle_submit_task:false, last_fetch_trigger_type:"LOG_LOOP_TH", this:0x2b3b5b6d4530}, [StateMgr]={this:0x2b3b5b6d82b0, self:"10.191.69.51:2882", palf_id:1008, role:"FOLLOWER", replica_type:"NORMAL_REPLICA", state:"ACTIVE", prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, leader:"10.186.69.176:2882", leader_epoch:0, is_sync_enabled:true, allow_vote:true, allow_vote_persisted:true, pending_end_lsn:{lsn:18446744073709551615}, scan_disk_log_finished:true, last_check_start_id:404884536, is_changing_config_with_arb:false, reconfirm_start_time_us:-1, palf_role_change_cb:0x2b3b5b6deb70, allow_vote:true}, [ConfigMgr]={palf_id:1008, self:"10.191.69.51:2882", alive_paxos_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, alive_paxos_replica_num:5, log_ms_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:1, config_seq:161}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:17206, lock_time:1716901298232120}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:162}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17206, lock_time:1716901298444615}}, prev_log_proposal_id:1, prev_lsn:{lsn:3278614736794}, prev_mode_pid:1}, running_args:{server:{server:"0.0.0.0:0", timestamp:-1, flag:0}, curr_member_list:0, curr_replica_num:0, new_replica_num:0, config_version:{proposal_id:9223372036854775807, config_seq:-1}, ref_scn:{val:18446744073709551615, v:3}, lock_owner:-1, lock_type:0, added_list:0, removed_list:0, type:"Invalid"}, state:0, checking_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, reconfig_barrier:{prev_log_proposal_id:9223372036854775807, prev_lsn:{lsn:0}, prev_end_lsn:{lsn:0}, prev_mode_pid:9223372036854775807}, persistent_config_version:{proposal_id:1, config_seq:162}, ms_ack_list:0, resend_config_version:{proposal_id:9223372036854775807, config_seq:-1}, resend_log_list:{learner_num:0, learner_array:[]}, last_submit_config_log_time_us:-1, need_change_config_bkgd:false, bkgd_config_version:{proposal_id:9223372036854775807, config_seq:-1}, region:"sys_region", register_time_us:1719306020110244, parent:"0.0.0.0:0", parent_keepalive_time_us:-1, last_submit_register_req_time_us:1719306020110244, children:{learner_num:0, learner_array:[]}, last_submit_keepalive_time_us:-1, this:0x2b3b5b6d5c70}, [ModeMgr]={palf_id:1008, self:"10.191.69.51:2882", applied_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543412178000, v:0}}, accepted_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543412178000, v:0}}, last_submit_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543412178000, v:0}}, state:"MODE_INIT", new_proposal_id:9223372036854775807, local_max_lsn:{lsn:18446744073709551615}, local_max_log_pid:9223372036854775807, max_majority_accepted_pid:9223372036854775807, max_majority_lsn:{lsn:18446744073709551615}, max_majority_accepted_mode_meta:{version:-1, proposal_id:9223372036854775807, mode_version:9223372036854775807, access_mode:0, ref_scn:{val:18446744073709551615, v:3}}, follower_list:0, ack_list:0, majority_cnt:9223372036854775807, last_submit_req_ts:-1, resend_mode_meta_list:{learner_num:0, learner_array:[]}}, [LogEngine]={palf_id:1008, is_inited:true, min_block_max_scn:{val:1718672313723025210, v:0}, min_block_id:78656, min_block_min_scn:{val:1718672310719159436, v:0}, base_lsn_for_block_gc:{lsn:5845831864320}, log_meta:{version:1, log_prepare_meta:{version:1, voted_for:"0.0.0.0:0", log_proposal_id:1}, log_config_meta:{version:3, proposal_id:1, prev:{version:1, config:{config_version:{proposal_id:1, config_seq:161}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:1, lock_owner:17206, lock_time:1716901298232120}}, curr:{version:1, config:{config_version:{proposal_id:1, config_seq:162}, log_sync_memberlist:5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, log_sync_replica_num:5, arbitration_member:{server:"0.0.0.0:0", timestamp:-1, flag:0}, learnerlist:{learner_num:0, learner_array:[]}, degraded_learnerlist:{learner_num:0, learner_array:[]}}, lock_meta:{version:1, lock_type:0, lock_owner:17206, lock_time:1716901298444615}}, prev_log_proposal_id:1, prev_lsn:{lsn:3278614736794}, prev_mode_pid:1}, log_snapshot_meta:{version:1, base_lsn:{lsn:5845831864320}, prev_log_info:{log_id:-1, lsn:{lsn:18446744073709551615}, scn:{val:18446744073709551615, v:3}, log_proposal_id:9223372036854775807, accum_checksum:-1}}, log_replica_property_meta:{allow_vote:true, replica_type:"NORMAL_REPLICA"}, log_mode_meta:{version:1, proposal_id:1, mode_version:0, access_mode:1, ref_scn:{val:1713332543412178000, v:0}}}, log_meta_storage:{log_tail:{lsn:1818624}, readable_log_tail:{lsn:1818624}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:5880994762752}, min_scn:{val:0, v:0}, curr_block_id:0, palf_id:1008}, block_mgr:{log_dir:"/data/clog/tenant_1004/1008/meta", dir_fd:857, min_block_id:0, max_block_id:1, curr_writable_block_id:0}, logical_block_size_:67104768, curr_block_writable_size_:65286144, block_header_serialize_buf_:0x2b3b5b6da438, flashback_version:0}, log_storage:{log_tail:{lsn:5880977363420}, readable_log_tail:{lsn:5880977363420}, log_block_header:{magic:18754, version:1, min_lsn:{lsn:5880927657984}, min_scn:{val:1719305457070694001, v:0}, curr_block_id:87638, palf_id:1008}, block_mgr:{log_dir:"/data/clog/tenant_1004/1008/log", dir_fd:861, min_block_id:78656, max_block_id:87639, curr_writable_block_id:87638}, logical_block_size_:67104768, curr_block_writable_size_:17399332, block_header_serialize_buf_:0x2b3b5b6dc0b8, flashback_version:0}, palf_epoch:4, last_purge_throttling_ts:-1, this:0x2b3b5b6d89f0}, [Reconfirm]={palf_id:1008, self:"10.191.69.51:2882", state:"INITED", new_proposal_id:9223372036854775807, prepare_log_ack_list:0, curr_paxos_follower_list:0, majority_cnt:0, majority_max_log_server:"0.0.0.0:0", majority_max_accept_pid:9223372036854775807, majority_max_lsn:{lsn:18446744073709551615}, saved_end_lsn:{lsn:18446744073709551615}, last_submit_prepare_req_time_us:-1, last_fetch_log_time_us:-1, last_record_sw_start_id:-1, last_notify_fetch_time_us:-1, last_purge_throttling_time_us:-1, this:0x2b3b5b6d8430}) [2024-06-25 17:00:23.912320] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=114][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1008, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:23.912326] INFO [PALF] check_and_switch_state (palf_handle_impl.cpp:2652) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=5] [PALF_DUMP](palf_id=1008, self="10.191.69.51:2882", ack_info_list=[]) [2024-06-25 17:00:23.912332] WDIAG [PALF] get_ack_info_array (log_sliding_window.cpp:2742) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=5][errcode=-4018] match_lsn_map_ get failed(ret=-4018, palf_id=1008, self="10.191.69.51:2882", tmp_server={server:"10.186.69.171:2882", timestamp:1, flag:0}) [2024-06-25 17:00:23.912335] INFO [PALF] report_log_task_trace (log_sliding_window.cpp:4290) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=3] current log_task status(palf_id=1008, self="10.191.69.51:2882", log_id=404884536, log_task={header:{begin_lsn:{lsn:18446744073709551615}, end_lsn:{lsn:18446744073709551615}, log_id:-1, min_scn:{val:18446744073709551615, v:3}, max_scn:{val:18446744073709551615, v:3}, data_len:0, proposal_id:9223372036854775807, prev_lsn:{lsn:18446744073709551615}, prev_proposal_id:9223372036854775807, committed_end_lsn:{lsn:18446744073709551615}, data_checksum:-1, accum_checksum:-1, is_padding_log:false, is_raw_write:false}, state_map:{val:0}, ref_cnt:0, gen_ts:-1, freeze_ts:-1, submit_ts:-1, flushed_ts:-1, gen_to_freeze cost time:0, gen_to_submit cost time:0, submit_to_flush cost time:0}, ack_info_list=[]) [2024-06-25 17:00:23.912972] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [62585][T1_Occam][T1][Y0-0000000000000000-0-0] [lt=15] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-06-25 17:00:23.914307] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.917012] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [100337][T1003_Occam][T1003][Y0-0000000000000000-0-0] [lt=11] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-06-25 17:00:23.918092] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.922651] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=92000, remain_us=20804524, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=92, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:23.923023] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [62718][T1_ReqMemEvict][T1][Y0-0000000000000000-0-0] [lt=11] sql audit evict task end(request_manager_->get_tenant_id()=1, evict_high_mem_level=969588866, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=409199, mem_used=769505280) [2024-06-25 17:00:23.923885] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.928682] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=5020585, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=195, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:23.929824] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=715516, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=223, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:23.932224] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:23.932264] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=14] [PALF STAT GROUP LOG INFO](palf_id=1005, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:23.932274] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:23.932277] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.932280] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.932283] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.932493] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325962BB5-0-0] [lt=12][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301560, send_timestamp=1719306001630933) [2024-06-25 17:00:23.933770] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.934945] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:23.936505] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=10][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4013, dropped:231, tid:12807}]) [2024-06-25 17:00:23.937904] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-000616430B73EF51-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=330, fly_ts=22301560, send_timestamp=1719306001636344) [2024-06-25 17:00:23.940683] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:23.940854] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [63346][T1001_Occam][T1001][Y0-0000000000000000-0-0] [lt=12] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}]) [2024-06-25 17:00:23.942563] INFO [COMMON] replace_fragment_node (ob_kvcache_map.cpp:696) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] Cache replace map node details(ret=0, replace_node_count=0, replace_time=0, replace_start_pos=60100000, replace_num=100000) [2024-06-25 17:00:23.942574] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=10] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=10) [2024-06-25 17:00:23.943390] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:23.943400] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:23.943406] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=4][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=29999958, timeout_timestamp=1719306053942986) [2024-06-25 17:00:23.943413] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:23.943416] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=3][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:23.943422] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=5][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:23.943425] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=2][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:23.943429] WDIAG [SQL] open (ob_result_set.cpp:157) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=4][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:23.943432] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=3][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:23.943435] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=3][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"SELECT ls_id,zone_priority,manual_leader_server,blacklist FROM __all_ls_election_reference_info where tenant_id=1002"}) [2024-06-25 17:00:23.943442] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=6][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"SELECT ls_id,zone_priority,manual_leader_server,blacklist FROM __all_ls_election_reference_info where tenant_id=1002"}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:23.943447] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=4][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:23.943454] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=6][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:23.943457] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=3][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:23.943460] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=2][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:23.943465] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63163][T1002_Occam][T1001][YB420ABF4533-00061661469F7F1C-0-0] [lt=3][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"SELECT ls_id,zone_priority,manual_leader_server,blacklist FROM __all_ls_election_reference_info where tenant_id=1002"}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:23.943471] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63163][T1002_Occam][T1002][YB420ABF4533-00061661469F7F1C-0-0] [lt=5][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"SELECT ls_id,zone_priority,manual_leader_server,blacklist FROM __all_ls_election_reference_info where tenant_id=1002"}, aret=-4013, ret=-4013) [2024-06-25 17:00:23.943474] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63163][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=SELECT ls_id,zone_priority,manual_leader_server,blacklist FROM __all_ls_election_reference_info where tenant_id=1002) [2024-06-25 17:00:23.943480] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63163][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:23.943483] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63163][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=2][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:23.943489] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63163][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] query failed(ret=-4013, conn=0x2b3a7c6da050, start=1719306023942987, sql=SELECT ls_id,zone_priority,manual_leader_server,blacklist FROM __all_ls_election_reference_info where tenant_id=1002) [2024-06-25 17:00:23.943494] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63163][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:23.943498] WDIAG get_my_sql_result_ (ob_table_access_helper.h:435) [63163][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] GCTX.sql_proxy_ read failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", MTL_ID()=1002, tenant_id=1001, columns=0x2b3a8cc56190, table=__all_ls_election_reference_info, condition=where tenant_id=1002, sql=SELECT ls_id,zone_priority,manual_leader_server,blacklist FROM __all_ls_election_reference_info where tenant_id=1002, columns_str="ls_id,zone_priority,manual_leader_server,blacklist") [2024-06-25 17:00:23.943507] WDIAG read_and_convert_to_tuples_ (ob_table_access_helper.h:379) [63163][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4013] fail to get ObMySQLResult(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", MTL_ID()=1002, table=__all_ls_election_reference_info, condition=where tenant_id=1002) [2024-06-25 17:00:23.943523] WDIAG [COORDINATOR] get_all_ls_election_reference_info (table_accessor.cpp:580) [63163][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] read multi row failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", all_ls_election_reference_info=[]) [2024-06-25 17:00:23.943530] WDIAG [COORDINATOR] refresh (ob_leader_coordinator.cpp:144) [63163][T1002_Occam][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4013] get all ls election reference info failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.944104] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=0] will sleep(sleep_us=4000, remain_us=29724784, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=4, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:23.945399] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [63469][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=5] ====== checkpoint timer task ====== [2024-06-25 17:00:23.945408] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:193) [63469][T1001_TxCkpt][T1001][Y0-0000000000000000-0-0] [lt=7] no logstream(ret=0, ls_cnt=0) [2024-06-25 17:00:23.951600] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.951609] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.951614] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.951621] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.951626] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.955713] INFO [SHARE] run_loop_ (ob_bg_thread_monitor.cpp:331) [62818][BGThreadMonitor][T0][Y0-0000000000000000-0-0] [lt=8] current monitor number(seq_=-1) [2024-06-25 17:00:23.958136] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BB7-0-0] [lt=4] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=91, avg_log_submit_to_slide_time=845) [2024-06-25 17:00:23.958143] INFO [PALF] read (log_cache.cpp:100) [99515][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=13] [PALF STAT HOT CACHE HIT RATE](palf_id=1008, read_size=1220, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:23.958515] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.958525] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.958530] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.958537] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.958541] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.958555] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e6b6780, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221522301, assoc_session_id:3221522301, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306023689605, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3a97f75ff8, id:3221522301, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e6b6780}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:23.962550] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-0006164325C30193-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf16f470, extra_info="") [2024-06-25 17:00:23.962650] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=82, this=0x2b3aaf1793b0, extra_info="palf_id:1010") [2024-06-25 17:00:23.962661] INFO [LIB] stat (utility.h:1214) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=5, this=0x2b3a8ce1d3b0, extra_info="") [2024-06-25 17:00:23.962671] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=5] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3aaf1795e0, extra_info="palf_id:1010") [2024-06-25 17:00:23.962852] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4538-00061669548FCCB0-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3aaf16f6a0, extra_info="palf_id:1010") [2024-06-25 17:00:23.962857] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4538-00061669548FCCB0-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=291, this=0x2b3aaf16f8d0, extra_info="palf_id:1010") [2024-06-25 17:00:23.963414] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:130) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=3] ====== checkpoint timer task ====== [2024-06-25 17:00:23.963900] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] succ to get rec scn(*this={this:0x2b3aae604030, ls_id:{id:1006}, tenant_id:1002, state:"L_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171811467, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:151}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:23.963915] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=15] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb4080, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171812500, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb4080, snapshot_version:{val:1718209171812500, v:0}, ls_id:{id:1006}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:23.963933] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=14] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1006}) [2024-06-25 17:00:23.963958] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3a9534a510, is_inited:true, freezing_scn:{val:1718285174571744002, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1006}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018752321001, v:0}, clog_base_lsn:{lsn:75425759232}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315392841000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:1, proposal_id:1, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1006}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1006, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:110, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:23.963993] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=33] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1006}) [2024-06-25 17:00:23.964011] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] get_binary_search_range_(ret=0, this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1112, max_block_id=1124, result_block_id=274894685184, committed_lsn={lsn:75469144824}, scn={val:1719306023690503001, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:23.964556] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=5] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759419, offset=21480812}, end_cursor:ObLogCursor{file_id=67, log_id=18759420, offset=21481053}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbf0050, len:2708, log_data_len:241, seq:18759419, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:23.964575] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:75425759232}, checkpoint_scn={val:1719306023690503001, v:0}, ls_id={id:1006}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:23.964590] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:75425759232}, new_base_lsn={lsn:75425759232}, lsn={lsn:75425759232}) [2024-06-25 17:00:23.964598] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:75425759232}, ls->get_ls_id()={id:1006}) [2024-06-25 17:00:23.965069] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] succ to get rec scn(*this={this:0x2b3aae6b6030, ls_id:{id:1007}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171811972, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:149}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:23.965079] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb4e90, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171813010, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb4e90, snapshot_version:{val:1718209171813010, v:0}, ls_id:{id:1007}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:23.965091] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=10] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1007}) [2024-06-25 17:00:23.965110] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3a86ce4510, is_inited:true, freezing_scn:{val:1718289975940695002, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1007}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018870127000, v:0}, clog_base_lsn:{lsn:85155950592}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315408583001, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1007}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1007, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:23.965135] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=24] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1007}) [2024-06-25 17:00:23.965143] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] get_binary_search_range_(ret=0, this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1257, max_block_id=1269, result_block_id=274894685184, committed_lsn={lsn:85187192783}, scn={val:1719306023872550000, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:23.965632] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=13] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759420, offset=21481053}, end_cursor:ObLogCursor{file_id=67, log_id=18759421, offset=21481294}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbec050, len:2467, log_data_len:241, seq:18759420, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:23.965643] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:85155950592}, checkpoint_scn={val:1719306023872550000, v:0}, ls_id={id:1007}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:23.965651] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:85155950592}, new_base_lsn={lsn:85155950592}, lsn={lsn:85155950592}) [2024-06-25 17:00:23.965657] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:85155950592}, ls->get_ls_id()={id:1007}) [2024-06-25 17:00:23.965841] INFO [STORAGE.TRANS] alloc (ob_ts_response_handler.cpp:108) [62236][pnio1][T0][YB420ABF4533-0006166171AF6607-0-0] [lt=10] ts response task statistics(alloc_count=2495631178, free_count=1247864980) [2024-06-25 17:00:23.965936] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=4] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1004, need_refresh=false, gts_local_cache={srr:[mts=1719306023465468], gts:1719306023465151000, latest_srr:[mts=1719306023965931]}) [2024-06-25 17:00:23.966127] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=3] succ to get rec scn(*this={this:0x2b3ab7604030, ls_id:{id:1008}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629001, v:0}, last_push_gc_task_ts_:1718209171812477, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:149}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:23.966133] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62235][pnio1][T0][YB420ABF4533-0006166171AF6606-0-0] [lt=8][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:23.966135] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb5ca0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171815569, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb5ca0, snapshot_version:{val:1718209171815569, v:0}, ls_id:{id:1008}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:23.966142] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62235][pnio1][T0][YB420ABF4533-0006166171AF6606-0-0] [lt=8] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:23.966144] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1008}) [2024-06-25 17:00:23.966154] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62235][pnio1][T0][YB420ABF4533-0006166171AF6606-0-0] [lt=5] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306023965759]}) [2024-06-25 17:00:23.966162] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3a86daa510, is_inited:true, freezing_scn:{val:1718286006014126001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1008}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018543922003, v:0}, clog_base_lsn:{lsn:74083663872}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315055094004, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1008}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1008, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-550728704, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:23.966186] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=23] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1008}) [2024-06-25 17:00:23.966193] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] get_binary_search_range_(ret=0, this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1092, max_block_id=1104, result_block_id=274894685184, committed_lsn={lsn:74143923162}, scn={val:1719306023866219000, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:23.966568] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=28000, remain_us=28064867, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=28, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:23.966618] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=7] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759421, offset=21481294}, end_cursor:ObLogCursor{file_id=67, log_id=18759422, offset=21481535}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbe0050, len:2226, log_data_len:241, seq:18759421, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:23.966629] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:74083663872}, checkpoint_scn={val:1719306023866219000, v:0}, ls_id={id:1008}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:23.966636] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:74083663872}, new_base_lsn={lsn:74083663872}, lsn={lsn:74083663872}) [2024-06-25 17:00:23.966642] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:74083663872}, ls->get_ls_id()={id:1008}) [2024-06-25 17:00:23.966729] INFO [STORAGE] exec_wait (ob_tmp_file_cache.cpp:1531) [42764][T1002_ComTimerT][T1002][Y0-0000000000000000-0-0] [lt=9] succeed to do one round of tmp block io(ret=-4023, loop_nums=0, wait_io_cnt=0, washing_count=0, block_cache_num=0, page_cache_num=0) [2024-06-25 17:00:23.967109] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=3] succ to get rec scn(*this={this:0x2b3ab76b6030, ls_id:{id:1009}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171812974, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:152}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:23.967117] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb6ab0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171817146, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb6ab0, snapshot_version:{val:1718209171817146, v:0}, ls_id:{id:1009}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:23.967125] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1009}) [2024-06-25 17:00:23.967143] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3aaf086510, is_inited:true, freezing_scn:{val:1718283438009682001, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1009}, ls_create_status:1, clog_checkpoint_scn:{val:1719306018805948000, v:0}, clog_base_lsn:{lsn:86632255488}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315224596003, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1009}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1009, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:0, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:23.967164] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=21] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="ALL_EMPTY", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]=NULL, min_rec_scn={val:4611686018427387903, v:0}, ls_id_={id:1009}) [2024-06-25 17:00:23.967171] INFO [PALF] get_binary_search_range_ (palf_handle_impl.cpp:1983) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] get_binary_search_range_(ret=0, this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, min_block_id=1279, max_block_id=1291, result_block_id=274894685184, committed_lsn={lsn:86651605311}, scn={val:1719306023909397000, v:0}, last_locate_scn={val:18446744073709551615, v:3}, last_locate_block=274894685184) [2024-06-25 17:00:23.967650] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [63258][T1002_OB_SLOG][T1002][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=67, log_id=18759422, offset=21481535}, end_cursor:ObLogCursor{file_id=67, log_id=18759423, offset=21481776}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbe4050, len:1985, log_data_len:241, seq:18759422, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:23.967661] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:209) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [CHECKPOINT] update clog checkpoint successfully(clog_checkpoint_lsn={lsn:86632255488}, checkpoint_scn={val:1719306023909397000, v:0}, ls_id={id:1009}, service_type="MAX_DECIDED_SCN") [2024-06-25 17:00:23.967668] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1009, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:86632255488}, new_base_lsn={lsn:86632255488}, lsn={lsn:86632255488}) [2024-06-25 17:00:23.967672] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:86632255488}, ls->get_ls_id()={id:1009}) [2024-06-25 17:00:23.968140] INFO [STORAGE.TRANS] get_rec_scn (ob_trans_ctx_mgr_v4.cpp:1412) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=4] succ to get rec scn(*this={this:0x2b3a84404030, ls_id:{id:1010}, tenant_id:1002, state:"F_WORKING", total_tx_ctx_count:0, active_tx_count:0, total_active_readonly_request_count:0, ls_retain_ctx_mgr:{retain_ctx_list_.get_size():0, max_wait_ckpt_ts_:{val:1718209168742629000, v:0}, last_push_gc_task_ts_:1718209171813475, skip_remove_cnt_:0}, aggre_rec_scn:{val:18446744073709551615, v:3}, prev_aggre_rec_scn:{val:18446744073709551615, v:3}, uref:151}, aggre_rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:23.968148] INFO [STORAGE.TRANS] get_rec_scn (ob_tx_ctx_memtable.cpp:182) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=8] tx ctx memtable get rec scn(this={ObITable:{this:0x2b3a86cb78c0, key:{tablet_id:{id:49401}, column_group_idx:0, table_type:"TX_CTX_MEMTABLE", scn_range:{start_scn:{val:1, v:0}, end_scn:{val:1718209171818018, v:0}}}, ref_cnt:2, upper_trans_version:-4007, timestamp:0}, this:0x2b3a86cb78c0, snapshot_version:{val:1718209171818018, v:0}, ls_id:{id:1010}, is_frozen:false}, rec_scn={val:4611686018427387903, v:0}) [2024-06-25 17:00:23.968157] INFO [STORAGE.TABLELOCK] get_rec_scn (ob_lock_memtable.cpp:790) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=7] rec_scn of ObLockMemtable is (rec_scn_={val:4611686018427387903, v:0}, flushed_scn_={val:0, v:0}, pre_rec_scn_={val:4611686018427387903, v:0}, freeze_scn_={val:0, v:0}, max_committed_scn_={val:18446744073709551615, v:3}, is_frozen_=false, ls_id_={id:1010}) [2024-06-25 17:00:23.968174] INFO [MDS] get_rec_scn (mds_table_mgr.cpp:260) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=6] [FREEZE]no valid mds table there, return MAX SCN(ret=0, ret="OB_SUCCESS", min_rec_scn={val:4611686018427387903, v:0}, order_flusher_for_one={high_priority_flusher:size:0, ordered_tablets:{}, *this={this:0x2b3aaf132510, is_inited:true, freezing_scn:{val:1718283437110618003, v:0}, ls:{ls_meta:{tenant_id:1002, ls_id:{id:1010}, ls_create_status:1, clog_checkpoint_scn:{val:1719198894656674000, v:0}, clog_base_lsn:{lsn:75627073536}, rebuild_seq:0, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:0}, replayable_point:{val:1716917967818310002, v:0}, tablet_change_checkpoint_scn:{val:1719287315483366000, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}}}, switch_epoch:0, log_handler:{role:2, proposal_id:0, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1010}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1010, proposal_id:9223372036854775807, role:2, parent:null, context:{issue_task_num:0, issue_version:-1, last_fetch_ts:-1, max_submit_lsn:{lsn:18446744073709551615}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:-12647163, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}}}) [2024-06-25 17:00:23.968198] INFO [STORAGE.TRANS] get_rec_scn (ob_ls_tx_service.cpp:524) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=22] [CHECKPOINT] ObLSTxService::get_rec_scn(common_checkpoint_type="TX_DATA_MEMTABLE_TYPE", common_checkpoints_[min_rec_scn_common_checkpoint_type_index]={ObIMemtableMgr:{Memtables:this:0x2b3aaf1321b0, ref_cnt:1, is_inited:true, tablet_id:{id:49402}, freezer:0x2b3aaf137cd0, table_type:1, memtable_head:1774, memtable_tail:1775, t3m:0x2b3a8a6b2030, tables:[null, null, null, null, null, null, null, null, null, null, null, null, null, null, 0x2b3a86c90080, null]}, is_freezing:false, ls_id:{id:1010}, mini_merge_recycle_commit_versions_ts:1719188582135344, tx_data_table:0x2b3aaf1391d0, ls_tablet_svr:0x2b3aaf132190, slice_allocator:0x2b3aaf139210}, min_rec_scn={val:1719198894656674000, v:0}, ls_id_={id:1010}) [2024-06-25 17:00:23.968212] INFO [STORAGE] update_clog_checkpoint (ob_checkpoint_executor.cpp:180) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=13] [CHECKPOINT] clog checkpoint no change(checkpoint_scn={val:1719198894656674000, v:0}, checkpoint_scn_in_ls_meta={val:1719198894656674000, v:0}, ls_id={id:1010}, service_type="TRANS_SERVICE") [2024-06-25 17:00:23.968218] WDIAG [PALF] set_base_lsn (palf_handle_impl.cpp:1554) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=0] no need to set new base lsn, curr base lsn is greater than or equal to new base lsn(this={palf_id:1010, self:"10.191.69.51:2882", has_set_deleted:false}, curr_base_lsn={lsn:75627073536}, new_base_lsn={lsn:75627073536}, lsn={lsn:75627073536}) [2024-06-25 17:00:23.968223] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:184) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=5] [CHECKPOINT] advance palf base lsn successfully(checkpoint_lsn={lsn:75627073536}, ls->get_ls_id()={id:1010}) [2024-06-25 17:00:23.968226] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:191) [63294][T1002_TxCkpt][T1002][Y0-0000000000000000-0-0] [lt=2] succeed to update_clog_checkpoint(ret=0, ls_cnt=5) [2024-06-25 17:00:23.969107] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:23.969113] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:23.969116] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:23.969120] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:23.969122] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:23.969327] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABDC060-000616432547F0B3-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301560, send_timestamp=1719306001667767) [2024-06-25 17:00:23.978240] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.978249] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:23.978255] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.978261] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.978265] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.978545] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9756278, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=164, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:23.979382] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [63159][T1002_Occam][T1002][YB420ABF4533-000616615CEF8F5E-0-0] [lt=8] for each ls(succ_num=5, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:23.979412] WDIAG [STORAGE] get_ls (ob_ls_service.cpp:916) [63468][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=10][errcode=-4719] get log stream fail(ret=-4719, ls_id={id:1}) [2024-06-25 17:00:23.979422] INFO [STORAGE.TRANS] self_check (ob_tenant_weak_read_cluster_service.cpp:833) [63468][T1001_TenantWea][T1001][Y0-0000000000000000-0-0] [lt=8] [WRS] [TENANT_WEAK_READ_SERVICE] [CLUSTER_SERVICE] [SELF_CHECK] done(ret=0, ret="OB_SUCCESS", tenant_id=1001, need_start_service=false, need_stop_service=false, need_change_leader=false, is_in_service()=false, can_update_version=false, cur_leader_epoch=0, start_service_tstamp_=0, error_count_for_change_leader_=0, last_error_tstamp_for_change_leader_=0) [2024-06-25 17:00:23.979620] INFO [COORDINATOR] refresh (ob_leader_coordinator.cpp:146) [99158][T1004_Occam][T1004][Y0-0000000000000000-0-0] [lt=30] refresh __all_ls_election_reference_info success(ret=0, ret="OB_SUCCESS", *new_all_ls_election_reference_info=[{1,0,False,{False,},False,False,True}, {1001,0,False,{False,},False,False,True}, {1002,1,False,{False,},False,False,True}, {1003,1,False,{False,},False,False,True}, {1004,1,False,{False,},False,False,True}, {1005,0,False,{False,},False,False,True}, {1006,1,False,{False,},False,False,True}, {1007,1,False,{False,},False,False,True}, {1008,1,False,{False,},False,False,True}, {1039,0,False,{False,},False,False,True}]) [2024-06-25 17:00:23.979751] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62235][pnio1][T1001][YB420ABF4533-00061661718F6562-0-0] [lt=6][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:23.985404] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:23.985413] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:23.985417] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:23.985424] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:23.985428] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:23.987930] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.001965] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=0] will sleep(sleep_us=5000, remain_us=29666922, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=5, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:24.002337] INFO [ARCHIVE] do_thread_task_ (ob_archive_fetcher.cpp:312) [100473][T1003_ArcFetche][T1003][YB420ABF4533-0006164249EE6725-0-0] [lt=8] ObArchiveFetcher is running(thread_index=0) [2024-06-25 17:00:24.021631] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABF4533-0006165A1D4F0F41-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620713, send_timestamp=1719305973400917) [2024-06-25 17:00:24.021676] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABF4533-0006165A1D4F0F41-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301566, send_timestamp=1719306001720110) [2024-06-25 17:00:24.021819] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABF4533-0006165A1D4F0F41-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620708, send_timestamp=1719305973401110) [2024-06-25 17:00:24.021872] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABF4533-0006165A1D4F0F41-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301552, send_timestamp=1719306001720320) [2024-06-25 17:00:24.025374] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289103233024, global_cache_size=552067284992, tenant_max_wash_size=86073457, tenant_min_wash_size=86073457, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:24.025986] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99936][T1004_L0_G0][T1004][YB420ABF4533-00061660991FA32B-0-0] [lt=8][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:24.026179] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABF4533-00061660991FA32B-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620708, send_timestamp=1719305973405471) [2024-06-25 17:00:24.026217] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62249][pnio1][T0][YB420ABF4533-00061660991FA32B-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301558, send_timestamp=1719306001724659) [2024-06-25 17:00:24.026381] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABF4533-00061660991FA32B-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620708, send_timestamp=1719305973405673) [2024-06-25 17:00:24.026427] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABF4533-00061660991FA32B-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301556, send_timestamp=1719306001724871) [2024-06-25 17:00:24.027684] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [63474][T1001_LSArchive][T1001][YB420ABF4533-000616423F8E6725-0-0] [lt=5] gc stale ls task succ [2024-06-25 17:00:24.028545] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BB8-0-0] [lt=7] [PALF STAT LOG TASK TIME](palf_id=1007, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=99, avg_log_submit_to_slide_time=850) [2024-06-25 17:00:24.028561] INFO [PALF] read (log_cache.cpp:100) [99266][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=12] [PALF STAT HOT CACHE HIT RATE](palf_id=1007, read_size=1211, hit_cnt=10, read_cnt=10, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:24.030252] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [99931][T1004_L0_G0][T1004][YB420ABF4533-00061660991FA32B-0-0] [lt=7][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:24.030404] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABF4533-00061660991FA32B-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620717, send_timestamp=1719305973409687) [2024-06-25 17:00:24.030600] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62245][pnio1][T0][YB420ABF4533-00061660991FA32B-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=50620711, send_timestamp=1719305973409888) [2024-06-25 17:00:24.030626] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABF4533-00061660991FA32B-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1472, fly_ts=22301560, send_timestamp=1719306001729066) [2024-06-25 17:00:24.032282] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:24.032315] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:24.032320] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.032323] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.032326] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.034998] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:24.036425] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [62681][T1_TenantMetaMe][T1][Y0-0000000000000000-0-0] [lt=24] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:832, free_obj_hold_cnt:430, allocator used:5017712, allocator total:5167232}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:117, free_obj_hold_cnt:63, allocator used:11806380, allocator total:11806380}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():166656, full allocator used:1600, full allocator total:232192}, tablets_mem=17205804, tablets_mem_limit=8079907220, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:28, free_obj_hold_cnt:931, allocator used:1902656, allocator total:1962240}, pending_cnt=0, wait_gc_count=0, tablet count=949) [2024-06-25 17:00:24.040738] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.048331] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=29000, remain_us=27983104, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=29, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:24.048364] INFO [PALF] try_recycle_blocks (palf_env_impl.cpp:788) [99170][T1004_PalfGC][T1004][Y0-0000000000000000-0-0] [lt=14] LOG_DISK_OPTION(disk_options_wrapper_={disk_opts_for_stopping_writing:{log_disk_size(MB):2073600, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, disk_opts_for_recycling_blocks:{log_disk_size(MB):2073600, log_disk_utilization_threshold(%):80, log_disk_utilization_limit_threshold(%):95, log_disk_throttling_percentage(%):60, log_disk_throttling_maximum_duration(s):7200, log_writer_parallelism:3}, status:1, cur_unrecyclable_log_disk_size(MB):500452, sequence:1}) [2024-06-25 17:00:24.050251] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1333) [2024-06-25 17:00:24.051402] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.051411] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.051416] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.051422] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.051427] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.051652] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8100FF6F2-0-0] [lt=10] [PALF STAT GROUP LOG INFO](palf_id=1, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=4178, avg_group_log_size=379) [2024-06-25 17:00:24.051741] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [62592][T1_IOWorker][T1][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=4178) [2024-06-25 17:00:24.052505] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A0B9-0-0] [lt=10] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a5d1db6a0, extra_info="palf_id:1") [2024-06-25 17:00:24.052512] INFO [LIB] stat (utility.h:1197) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A0B9-0-0] [lt=6] [PALF STAT LOG LIFE TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=852, this=0x2b3a5d1db8d0, extra_info="palf_id:1") [2024-06-25 17:00:24.052574] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62747][T1_ReplaySrv4][T1][Y0-0000000000000000-0-0] [lt=9] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:24.053183] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [99392][T1004_ReqMemEvi][T1004][Y0-0000000000000000-0-0] [lt=7] sql audit evict task end(request_manager_->get_tenant_id()=1004, evict_high_mem_level=20669530113, evict_high_size_level=9000000, evict_batch_count=0, elapse_time=0, size_used=8345654, mem_used=15148855296) [2024-06-25 17:00:24.053848] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63292][T1002_ReqMemEvi][T1002][Y0-0000000000000000-0-0] [lt=8] sql audit evict task end(request_manager_->get_tenant_id()=1002, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7318, mem_used=12478464) [2024-06-25 17:00:24.057432] INFO [LIB] stat (utility.h:1214) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=20, stat_interval=1000000, avg (count/cost)=6, this=0x2b3afcbfec70, extra_info="") [2024-06-25 17:00:24.059052] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=17765926, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=112, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:24.060877] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=1] will sleep(sleep_us=6000, remain_us=29608010, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=6, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:24.062909] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301566, send_timestamp=1719306001761342) [2024-06-25 17:00:24.063655] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.063664] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.063669] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:24.063676] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.063681] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.064796] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1BE9-0-0] [lt=3] [PALF STAT GROUP LOG INFO](palf_id=1009, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=9, avg_log_batch_cnt=0, total_group_log_size=1098, avg_group_log_size=122) [2024-06-25 17:00:24.065638] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0B6-0-0] [lt=8] [PALF STAT LOG TASK TIME](palf_id=1009, self="10.191.69.51:2882", total_slide_log_cnt=9, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=89, avg_log_submit_to_slide_time=848) [2024-06-25 17:00:24.065644] INFO [PALF] read (log_cache.cpp:100) [63318][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=12] [PALF STAT HOT CACHE HIT RATE](palf_id=1009, read_size=1098, hit_cnt=9, read_cnt=9, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:24.066523] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.066533] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:24.066538] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:24.066545] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.066550] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.068452] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=93000, remain_us=20658723, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=93, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:24.069135] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.069142] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.069144] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.069148] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:24.069150] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:24.070417] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.070425] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:24.070429] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:24.070435] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.070440] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.076319] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.076327] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.076332] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.076338] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.076342] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.078158] WDIAG setup_next_scanner (ob_direct_receive_op.cpp:204) [62907][TimezoneMgr][T1002][YB420ABF4533-0006166101CF5DAC-0-0] [lt=4][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:24.078172] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:103) [62907][TimezoneMgr][T1002][YB420ABF4533-0006166101CF5DAC-0-0] [lt=0] already timeout, do not need sleep(sleep_us=0, remain_us=27855452, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=0, timeout_timestamp=1719306051933624) [2024-06-25 17:00:24.078189] INFO pktc_resp_cb_on_msg (pktc_resp.h:70) [62247][pnio1][T0][Y0-0000000000000000-0-0] [lt=8] PNIO resp cb not found: packet_id=1035878999 [2024-06-25 17:00:24.079808] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62241][pnio1][T1001][YB420ABF4533-00061661718F6563-0-0] [lt=8][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:24.082462] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=4866806, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=196, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:24.083584] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=561756, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=224, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:24.083729] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.083738] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.083743] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.083749] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.083754] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.086162] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.086171] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.086175] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.086181] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.086186] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.087988] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.088658] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.088666] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:24.088670] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.088675] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.088681] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.092833] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95800][T1004_L0_G1][T1004][YB420ABF4538-000616676A7FCC2D-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1006, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=11, avg_log_batch_cnt=0, total_group_log_size=18604, avg_group_log_size=1691) [2024-06-25 17:00:24.094501] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.094509] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:24.094513] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.094519] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.094524] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.100364] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=5] ====== check clog disk timer task ====== [2024-06-25 17:00:24.100380] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=9] get_disk_usage(ret=0, capacity(MB):=2073600, used(MB):=1658818) [2024-06-25 17:00:24.100394] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [99396][T1004_CKClogDis][T1004][Y0-0000000000000000-0-0] [lt=6] cannot_recycle_log_size statistics(cannot_recycle_log_size=524426718733, threshold=652298158080, need_update_checkpoint_scn=false) [2024-06-25 17:00:24.104059] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.104066] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:24.104071] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.104077] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.104081] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.120433] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.120441] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.120446] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.120451] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.120456] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.120771] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=1] will sleep(sleep_us=7000, remain_us=29548116, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=7, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:24.125013] WDIAG [STORAGE.TRANS] process_ (ob_gti_rpc.h:178) [62242][pnio1][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=0] gti rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:24.127745] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95800][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBBA6-0-0] [lt=8] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:24.128879] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0B7-0-0] [lt=6] [PALF STAT LOG TASK TIME](palf_id=1008, self="10.191.69.51:2882", total_slide_log_cnt=10, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=91, avg_log_submit_to_slide_time=842) [2024-06-25 17:00:24.129549] INFO [MDS] for_each_ls_in_tenant (mds_tenant_service.cpp:325) [63342][T1001_Occam][T1001][YB420ABF4533-000616615CFF8F5E-0-0] [lt=9] for each ls(succ_num=0, ret=0, ret="OB_SUCCESS") [2024-06-25 17:00:24.131117] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=1] will sleep(sleep_us=30000, remain_us=27900318, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=30, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:24.132327] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT GROUP LOG INFO](palf_id=1039, self="10.191.69.51:2882", role="LEADER", total_group_log_cnt=10, avg_log_batch_cnt=1, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:24.132337] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT APPEND DATA SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1211) [2024-06-25 17:00:24.132345] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:24.132360] INFO eloop_run (eloop.c:143) [62232][pnio1][T0][Y0-0000000000000000-0-0] [lt=7] PNIO [ratelimit] time: 1719306024132359, bytes: 29137869503639, bw: 0.730321 MB/s, add_ts: 1000249, add_bytes: 765988 [2024-06-25 17:00:24.132372] INFO [PALF] submit_log (palf_handle_impl.cpp:456) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=4] [PALF STAT APPEND DATA SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, append size=1211) [2024-06-25 17:00:24.132373] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9602451, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=165, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:24.132378] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:24.132381] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.132387] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.132389] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.132453] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1005, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1211) [2024-06-25 17:00:24.132563] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62236][pnio1][T0][YB420ABDC060-0006164325962BBF-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301560, send_timestamp=1719306001831002) [2024-06-25 17:00:24.132627] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC060-0006164325962BC0-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301558, send_timestamp=1719306001831069) [2024-06-25 17:00:24.135023] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:24.137906] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.137915] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.137919] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.137925] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.137930] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.140791] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.142637] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=5] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=9) [2024-06-25 17:00:24.142750] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62432][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142776] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62421][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142780] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62430][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142821] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62429][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142822] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62418][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142865] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62417][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142875] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62423][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142890] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62424][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142914] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62416][RpcIO][T0][Y0-0000000000000000-0-0] [lt=5] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142916] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62431][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142980] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62420][RpcIO][T0][Y0-0000000000000000-0-0] [lt=5] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142982] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62425][RpcIO][T0][Y0-0000000000000000-0-0] [lt=8] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.142982] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62415][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.143008] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62427][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.143041] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62426][RpcIO][T0][Y0-0000000000000000-0-0] [lt=9] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.143133] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62422][RpcIO][T0][Y0-0000000000000000-0-0] [lt=10] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.143345] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62419][RpcIO][T0][Y0-0000000000000000-0-0] [lt=7] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.143620] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62428][RpcIO][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.150159] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF79B4-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86d1d470, extra_info="") [2024-06-25 17:00:24.150264] INFO [LIB] stat (utility.h:1197) [63172][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=81, this=0x2b3a86d273b0, extra_info="palf_id:1007") [2024-06-25 17:00:24.150289] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=9, this=0x2b3a86d275e0, extra_info="palf_id:1007") [2024-06-25 17:00:24.150478] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0B8-0-0] [lt=6] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3a86d1d6a0, extra_info="palf_id:1007") [2024-06-25 17:00:24.150483] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0B8-0-0] [lt=4] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=285, this=0x2b3a86d1d8d0, extra_info="palf_id:1007") [2024-06-25 17:00:24.154316] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.154325] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.154330] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.154335] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.154339] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.156302] INFO [RPC.OBMYSQL] sm_conn_build_handshake (obsm_conn_callback.cpp:121) [62480][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=7] new mysql sessid created(conn.sessid_=3221583376, support_ssl=false) [2024-06-25 17:00:24.156319] INFO [RPC.OBMYSQL] init (obsm_conn_callback.cpp:141) [62480][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=8] sm conn init succ(conn.sessid_=3221583376, sess.client_addr_="10.191.69.51:48078") [2024-06-25 17:00:24.156326] INFO [RPC.OBMYSQL] do_accept_one (ob_sql_nio.cpp:1024) [62480][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=5] accept one succ(*s={this:0x2b3a307fc030, session_id:3221583376, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:0, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:24.156343] INFO [RPC.OBMYSQL] print_session_info (ob_sql_nio.cpp:1105) [62480][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=13] [sql nio session](*s={this:0x2b3a441f8030, session_id:3221510484, trace_id:YB420ABF4533-00061661700E6767-0-0, sql_handling_stage:11, sql_initiative_shutdown:false, reader:{fd:920}, err:0, last_decode_time:1719305944310019, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:1139, pending_flag:1, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:24.156542] WDIAG [RPC.OBMYSQL] handle_sock_event (ob_sql_nio.cpp:943) [62480][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=7][errcode=0] socket closed, it maybe disconnected by the client or by observer actively(mask=8197, *s={this:0x2b3a307fc030, session_id:3221583376, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:24.156552] INFO [RPC.OBMYSQL] on_disconnect (obsm_conn_callback.cpp:268) [62480][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=8] kill and revert session(conn.sessid_=3221583376, proxy_sessid=0, server_id=1, ret=0) [2024-06-25 17:00:24.156556] INFO [RPC.OBMYSQL] handle_pending_destroy_list (ob_sql_nio.cpp:918) [62480][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=4] sock ref clean, do destroy(*s={this:0x2b3a307fc030, session_id:3221583376, trace_id:Y0-0000000000000000-0-0, sql_handling_stage:-1, sql_initiative_shutdown:false, reader:{fd:869}, err:5, last_decode_time:0, pending_write_task:{buf:null, sz:0}, need_epoll_trigger_write:false, consume_size:0, pending_flag:0, may_handling_flag:true, handler_close_flag:false}) [2024-06-25 17:00:24.156563] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:223) [62480][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=7] mark session id unused(conn.sessid_=3221583376) [2024-06-25 17:00:24.156567] INFO [RPC.OBMYSQL] destroy (obsm_conn_callback.cpp:243) [62480][sql_nio1][T0][Y0-0000000000000000-0-0] [lt=3] connection close(sessid=3221583376, proxy_sessid=0, tenant_id=0, server_id=1, from_proxy=false, from_java_client=false, c/s protocol="OB_MYSQL_CS_TYPE", is_need_clear_sessid_=true, is_sess_alloc_=false, ret=0, trace_id=Y0-0000000000000000-0-0, conn.pkt_rec_wrapper_=[start_pkt_pos_:0, cur_pkt_pos_:0], disconnect_state=0) [2024-06-25 17:00:24.157408] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95800][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABF44-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1008, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:24.161133] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=10][errcode=-4383] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:24.162332] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABF4538-000616694DDF5240-0-0] [lt=9] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1006}}) [2024-06-25 17:00:24.162349] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABF4538-000616694DDF5240-0-0] [lt=10] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306024092344000, v:0}, arg={tenant_id:1004, ls_id:{id:1006}}, result={tenant_id:1004, ls_id:{id:1006}, cur_readable_scn:{val:1719306024092344000, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:24.162676] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABDC060-0006164325C3019B-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301568, send_timestamp=1719306001861108) [2024-06-25 17:00:24.162956] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABF4533-00061661728FC9A0-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301553, send_timestamp=1719306001861402) [2024-06-25 17:00:24.164197] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=9] re_register_parent reason(palf_id=1, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306020162212, last_first_register_time_us=1713323567951674, register_time_us=1719306020162212, curr_time_us=1719306024164195) [2024-06-25 17:00:24.164222] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=11] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1, self="10.191.69.51:2882", reason=3, leader="10.186.69.171:2882", register_time_us=1719306024164209, last_submit_register_req_time_us=1719306024164209, register_parent_reason=3) [2024-06-25 17:00:24.164230] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [62596][T1_LogLoop][T1][Y0-0000000000000000-0-0] [lt=7] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1, self="10.191.69.51:2882") [2024-06-25 17:00:24.169226] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.169236] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=10] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.169239] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.169242] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:24.169253] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=11] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:24.169317] INFO [LIB] stat (utility.h:1214) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=9] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=18, stat_interval=1000000, avg (count/cost)=6, this=0x2b3a8ce1cc70, extra_info="") [2024-06-25 17:00:24.169446] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-000616432547F0BA-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301556, send_timestamp=1719306001867890) [2024-06-25 17:00:24.172582] WDIAG [CLOG] open_palf (ob_log_service.cpp:472) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4018] failed to get palf_handle(ret=-4018, id={id:1}) [2024-06-25 17:00:24.172591] WDIAG [CLOG] get_palf_role (ob_log_service.cpp:527) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8][errcode=-4719] failed to open palf(ret=-4719, id={id:1}) [2024-06-25 17:00:24.173278] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1308, ret=-4013) [2024-06-25 17:00:24.173288] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:24.173295] WDIAG [SQL.EXE] task_execute_v2 (ob_executor_rpc_impl.cpp:143) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=4][errcode=-4013] rpc task_execute fail(ret=-4013, tenant_id=1001, svr="10.191.69.52:2882", timeout=1999670, timeout_timestamp=1719306026172600) [2024-06-25 17:00:24.173302] WDIAG task_execute_v2 (ob_executor_rpc_impl.cpp:152) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=6][errcode=-4013] [2024-06-25 17:00:24.173304] WDIAG [SQL.EXE] execute_with_sql (ob_remote_scheduler.cpp:250) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=2][errcode=-4013] task execute failed(ret=-4013) [2024-06-25 17:00:24.173309] WDIAG [SQL.EXE] schedule (ob_remote_scheduler.cpp:53) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=4][errcode=-4013] execute with sql failed(ret=-4013) [2024-06-25 17:00:24.173314] WDIAG [SQL] do_open_plan (ob_result_set.cpp:549) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=4][errcode=-4013] fail execute plan(ret=-4013) [2024-06-25 17:00:24.173317] WDIAG [SQL] open (ob_result_set.cpp:157) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=3][errcode=-4013] execute plan failed(ret=-4013) [2024-06-25 17:00:24.173321] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=3][errcode=-4013] open result set failed(ret=-4013) [2024-06-25 17:00:24.173324] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=3][errcode=-4013] result set open failed(ret=-4013, executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}) [2024-06-25 17:00:24.173331] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=7][errcode=-4013] execute failed(ret=-4013, tenant_id=1001, executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, retry_cnt=0, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:24.173337] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=5][errcode=-4013] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:0, local_retry_times:0, err_:-4013, err_:"OB_ALLOCATE_MEMORY_FAILED", retry_type:0, client_ret:-4013}, need_retry=false) [2024-06-25 17:00:24.173345] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=6][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:24.173347] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=2][errcode=-4013] result set close failed(ret=-4013) [2024-06-25 17:00:24.173350] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=3][errcode=-4013] failed to close result(close_ret=-4013, ret=-4013) [2024-06-25 17:00:24.173356] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63182][T1002_TenantInf][T1001][YB420ABF4533-000616616F6E9D3E-0-0] [lt=3][errcode=-4013] failed to process record(executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, record_ret=-4013, ret=-4013) [2024-06-25 17:00:24.173361] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:984) [63182][T1002_TenantInf][T1002][YB420ABF4533-000616616F6E9D3E-0-0] [lt=4][errcode=-4013] failed to process final(executor={ObIExecutor:, sql:"select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 "}, aret=-4013, ret=-4013) [2024-06-25 17:00:24.173365] WDIAG [SERVER] execute_read_inner (ob_inner_sql_connection.cpp:1768) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] execute sql failed(ret=-4013, tenant_id=1001, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:24.173370] WDIAG [SERVER] retry_while_no_tenant_resource (ob_inner_sql_connection.cpp:1043) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5][errcode=-4013] retry_while_no_tenant_resource failed(ret=-4013, tenant_id=1001) [2024-06-25 17:00:24.173374] WDIAG [SERVER] execute_read (ob_inner_sql_connection.cpp:1708) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=3][errcode=-4013] execute_read failed(ret=-4013, cluster_id=1713323324, tenant_id=1001) [2024-06-25 17:00:24.173379] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:131) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] query failed(ret=-4013, conn=0x2b3a7c6db190, start=1719306024172878, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:24.173384] WDIAG [COMMON.MYSQLP] read (ob_mysql_proxy.cpp:66) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=4][errcode=-4013] read failed(ret=-4013) [2024-06-25 17:00:24.173387] WDIAG [SHARE] load_pure_tenant_info_ (ob_tenant_info_proxy.cpp:413) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=2][errcode=-4013] failed to read(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", exec_tenant_id=1001, sql=select ORA_ROWSCN, * from __all_tenant_info where tenant_id = 1002 ) [2024-06-25 17:00:24.173410] WDIAG [SHARE] load_tenant_info (ob_tenant_info_proxy.cpp:368) [63182][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6][errcode=-4013] failed to load purge tenant info(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", tenant_id=1002, for_update=false) [2024-06-25 17:00:24.174420] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [63309][T1002_HAService][T1002][Y0-0000000000000000-0-0] [lt=9] start do ls ha handler(ls_id_array_=[{id:1010}, {id:1008}, {id:1007}, {id:1006}, {id:1009}]) [2024-06-25 17:00:24.179820] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62244][pnio1][T1001][YB420ABF4533-00061661718F6564-0-0] [lt=9][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:24.181334] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.181344] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:24.181349] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:24.181356] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.181362] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.181658] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=0] will sleep(sleep_us=8000, remain_us=29487229, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=8, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:24.182255] INFO [STORAGE] operator() (ob_tenant_freezer.cpp:140) [100368][T1003_Occam][T1003][Y0-0000000000000000-0-0] [lt=10] ====== tenant freeze timer task ====== [2024-06-25 17:00:24.182268] WDIAG [STORAGE] get_tenant_tx_data_mem_used_ (ob_tenant_freezer.cpp:612) [100368][T1003_Occam][T1003][Y0-0000000000000000-0-0] [lt=7][errcode=0] [TenantFreezer] no logstream(ret=0, ret="OB_SUCCESS", ls_cnt=0, tenant_info_={slow_freeze:false, slow_freeze_timestamp:0, freeze_interval:0, last_freeze_timestamp:0, slow_tablet:{id:0}}) [2024-06-25 17:00:24.188044] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.202430] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=26] get_max_applied_scn(scn={val:1713323697228952000, v:0}, this={ls_id_:{id:1008}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1713323697228952000, v:0}}) [2024-06-25 17:00:24.202443] INFO [CLOG] get_min_unreplayed_log_info (ob_replay_status.cpp:1036) [63293][T1002_TenantWea][T1002][Y0-0000000000000000-0-0] [lt=10] get_min_unreplayed_log_info(lsn={lsn:74143923406}, scn={val:1719306024049345001, v:0}, this={ls_id_:{id:1008}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306024128878, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:74143923406}, next_to_submit_scn_:{val:1719306024049345001, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713323697228952000, v:0}, iterator_:{iterator_impl:{buf_:0x2b3bda663000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:74143923284}, end_lsn:{lsn:74143923406}, read_buf:{buf_len_:73728, buf_:0x2b3bda663000}, block_size:67104768, log_storage_:0x2b3a86ded470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306024049345000, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306024049345000, v:0}, data_checksum:3647337712, flag:0}}, init_mode_version:0, accumulate_checksum:3754208467, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306024049345000, v:0}}}}}) [2024-06-25 17:00:24.203608] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.203617] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.203621] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:24.203628] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.203632] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.204069] INFO [PALF] log_loop_ (log_loop_thread.cpp:155) [63177][T1002_LogLoop][T1002][Y0-0000000000000000-0-0] [lt=4] LogLoopThread round_cost_time(us)(round_cost_time=12) [2024-06-25 17:00:24.206874] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95515][T1_L0_G1][T1][YB420ABA45AB-000616C8107FC0C8-0-0] [lt=3] [PALF STAT COMMITTED LOG SIZE](palf_id=1, self="10.191.69.51:2882", committed size=4178) [2024-06-25 17:00:24.208418] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1961) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(dag_cnt=0, map_size=0) [2024-06-25 17:00:24.208428] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1971) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=9] dump_dag_status(running_dag_net_map_size=0, blocking_dag_net_list_size=0) [2024-06-25 17:00:24.208432] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_COMPACTION_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:24.208440] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=8] dump_dag_status(priority="PRIO_HA_HIGH", low_limit=8, up_limit=8, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:24.208443] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_COMPACTION_MID", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:24.208447] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_HA_MID", low_limit=5, up_limit=5, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:24.208451] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(priority="PRIO_COMPACTION_LOW", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:24.208454] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_HA_LOW", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:24.208457] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_DDL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:24.208460] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_DDL_HIGH", low_limit=6, up_limit=6, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:24.208463] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1980) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(priority="PRIO_TTL", low_limit=2, up_limit=2, running_task=0, ready_dag_count=0, waiting_dag_count=0) [2024-06-25 17:00:24.208467] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:24.208473] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:0, sys_task_type:3, dag_type_str:"MINI_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:24.208478] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:24.208480] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:2, sys_task_type:5, dag_type_str:"MINOR_EXECUTE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:24.208483] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:24.208487] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:4, sys_task_type:6, dag_type_str:"MAJOR_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:24.208489] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:24.208492] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:4, dag_type_str:"TX_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:24.208495] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:24.208501] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:4, sys_task_type:7, dag_type_str:"WRITE_CKPT", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:24.208503] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, dag_count=0) [2024-06-25 17:00:24.208507] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:0, sys_task_type:19, dag_type_str:"MDS_TABLE_MERGE", dag_module_str:"COMPACTION"}, scheduled_task_count=0) [2024-06-25 17:00:24.208511] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:24.208514] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"DDL", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:24.208516] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:24.208519] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"UNIQUE_CHECK", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:24.208522] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:24.208524] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:6, sys_task_type:2, dag_type_str:"SQL_BUILD_INDEX", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:24.208527] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, dag_count=0) [2024-06-25 17:00:24.208531] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:7, sys_task_type:12, dag_type_str:"DDL_KV_MERGE", dag_module_str:"DDL"}, scheduled_task_count=0) [2024-06-25 17:00:24.208534] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208537] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208540] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208543] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208547] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208551] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_COMPLETE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208553] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208557] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208561] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208564] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208566] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208569] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"SYS_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208574] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208576] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208579] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208581] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"DATA_TABLETS_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208585] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208588] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"TABLET_GROUP_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208591] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208594] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"MIGRATION_FINISH", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208598] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208601] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"INITIAL_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208604] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208607] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"START_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208610] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208613] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:1, dag_type_str:"FINISH_PREPARE_MIGRATION", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208617] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208620] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:3, sys_task_type:1, dag_type_str:"FAST_MIGRATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208623] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, dag_count=0) [2024-06-25 17:00:24.208626] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:1, dag_type_str:"VALIDATE", dag_module_str:"MIGRATE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208632] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=6] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-06-25 17:00:24.208635] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"TABLET_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-06-25 17:00:24.208638] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, dag_count=0) [2024-06-25 17:00:24.208642] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:16, dag_type_str:"FINISH_BACKFILL_TX", dag_module_str:"BACKFILL_TX"}, scheduled_task_count=0) [2024-06-25 17:00:24.208645] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:24.208647] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_META", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:24.208650] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:24.208655] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_PREPARE", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:24.208657] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:24.208660] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_FINISH", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:24.208663] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:24.208668] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_DATA", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:24.208670] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:24.208673] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"PREFETCH_BACKUP_INFO", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:24.208676] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:24.208680] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_INDEX_REBUILD", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:24.208683] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:24.208685] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:8, dag_type_str:"BACKUP_COMPLEMENT_LOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:24.208688] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:24.208692] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:5, sys_task_type:10, dag_type_str:"BACKUP_BACKUPSET", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:24.208695] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, dag_count=0) [2024-06-25 17:00:24.208698] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:5, sys_task_type:11, dag_type_str:"BACKUP_ARCHIVELOG", dag_module_str:"BACKUP"}, scheduled_task_count=0) [2024-06-25 17:00:24.208700] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:24.208703] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208707] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:24.208710] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208713] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:24.208718] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"SYS_TABLETS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208722] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:24.208724] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"DATA_TABLETS_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208727] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:24.208731] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_GROUP_META_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208735] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:24.208737] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_LS_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208740] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:24.208743] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"INITIAL_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208746] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:24.208749] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"START_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208754] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:24.208757] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"FINISH_TABLET_GROUP_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208760] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, dag_count=0) [2024-06-25 17:00:24.208762] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:14, dag_type_str:"TABLET_RESTORE", dag_module_str:"RESTORE"}, scheduled_task_count=0) [2024-06-25 17:00:24.208767] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=5] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, dag_count=0) [2024-06-25 17:00:24.208770] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:5, sys_task_type:15, dag_type_str:"BACKUP_CLEAN", dag_module_str:"BACKUP_CLEAN"}, scheduled_task_count=0) [2024-06-25 17:00:24.208773] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, dag_count=0) [2024-06-25 17:00:24.208776] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:3, sys_task_type:17, dag_type_str:"REMOVE_MEMBER", dag_module_str:"REMOVE_MEMBER"}, scheduled_task_count=0) [2024-06-25 17:00:24.208780] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, dag_count=0) [2024-06-25 17:00:24.208783] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_BACKFILL_TX", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-06-25 17:00:24.208786] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, dag_count=0) [2024-06-25 17:00:24.208789] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:1, sys_task_type:18, dag_type_str:"TRANSFER_REPLACE_TABLE", dag_module_str:"TRANSFER"}, scheduled_task_count=0) [2024-06-25 17:00:24.208792] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1983) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, dag_count=0) [2024-06-25 17:00:24.208796] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1984) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=4] dump_dag_status(type={init_dag_prio:8, sys_task_type:20, dag_type_str:"TTL_DELTE_DAG", dag_module_str:"TTL"}, scheduled_task_count=0) [2024-06-25 17:00:24.208799] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_MIGRATION", dag_net_count=0) [2024-06-25 17:00:24.208802] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_PREPARE_MIGRATION", dag_net_count=0) [2024-06-25 17:00:24.208804] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_COMPLETE_MIGRATION", dag_net_count=0) [2024-06-25 17:00:24.208807] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER", dag_net_count=0) [2024-06-25 17:00:24.208809] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_BACKUP", dag_net_count=0) [2024-06-25 17:00:24.208812] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_RESTORE", dag_net_count=0) [2024-06-25 17:00:24.208814] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=2] dump_dag_status[DAG_NET](type="DAG_NET_TYPE_BACKUP_CLEAN", dag_net_count=0) [2024-06-25 17:00:24.208817] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1988) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status[DAG_NET](type="DAG_NET_TRANSFER_BACKFILL_TX", dag_net_count=0) [2024-06-25 17:00:24.208820] INFO [COMMON] dump_dag_status (ob_dag_scheduler.cpp:1991) [63429][T1001_DagSchedu][T1001][Y0-0000000000000000-0-0] [lt=3] dump_dag_status(total_worker_cnt=43, total_running_task_cnt=0, work_thread_num=43, scheduled_task_cnt=0) [2024-06-25 17:00:24.212330] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=12] re_register_parent reason(palf_id=1008, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306020110244, last_first_register_time_us=1713332561478106, register_time_us=1719306020110244, curr_time_us=1719306024212330) [2024-06-25 17:00:24.212352] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=11] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882", reason=1, leader="10.186.69.176:2882", register_time_us=1719306024212342, last_submit_register_req_time_us=1719306024212342, register_parent_reason=1) [2024-06-25 17:00:24.212360] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=7] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1008, self="10.191.69.51:2882") [2024-06-25 17:00:24.212365] INFO [PALF] check_parent_health (log_config_mgr.cpp:2744) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=3] re_register_parent reason(palf_id=1007, self="10.191.69.51:2882", is_registering_timeout=true, first_registration=false, parent_timeout=false, parent_keepalive_time_us=-1, last_submit_register_req_time_us=1719306020110277, last_first_register_time_us=1713332560477990, register_time_us=1719306020110277, curr_time_us=1719306024212365) [2024-06-25 17:00:24.212377] INFO [PALF] register_parent_ (log_config_mgr.cpp:2601) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=6] register_parent_(ret=0, ret="OB_SUCCESS", palf_id=1007, self="10.191.69.51:2882", reason=1, leader="10.186.69.171:2882", register_time_us=1719306024212371, last_submit_register_req_time_us=1719306024212371, register_parent_reason=1) [2024-06-25 17:00:24.212382] INFO [PALF] check_parent_health (log_config_mgr.cpp:2753) [99172][T1004_LogLoop][T1004][Y0-0000000000000000-0-0] [lt=5] re register_parent success(ret=0, ret="OB_SUCCESS", palf_id=1007, self="10.191.69.51:2882") [2024-06-25 17:00:24.212634] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=17612344, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=113, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:24.214732] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=31000, remain_us=27816703, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=31, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:24.215362] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=94000, remain_us=20511814, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=94, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:24.220142] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.220151] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.220156] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:24.220162] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.220167] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.225603] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=13] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289107427328, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:24.228100] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC969-0-0] [lt=2] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3a86de7470, extra_info="") [2024-06-25 17:00:24.228112] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8109FC969-0-0] [lt=9] [PALF STAT GROUP LOG INFO](palf_id=1008, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1220, avg_group_log_size=122) [2024-06-25 17:00:24.228194] INFO [LIB] stat (utility.h:1197) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=72, this=0x2b3a86df13b0, extra_info="palf_id:1008") [2024-06-25 17:00:24.228206] INFO [LIB] stat (utility.h:1214) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=8] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3a8ce1daf0, extra_info="") [2024-06-25 17:00:24.228216] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=7] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=9, this=0x2b3a86df15e0, extra_info="palf_id:1008") [2024-06-25 17:00:24.228953] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0BB-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3a86de76a0, extra_info="palf_id:1008") [2024-06-25 17:00:24.228958] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0BB-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=845, this=0x2b3a86de78d0, extra_info="palf_id:1008") [2024-06-25 17:00:24.231130] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-0006164323C5053F-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5386, fly_ts=22301559, send_timestamp=1719306001929571) [2024-06-25 17:00:24.232385] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:24.232419] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:24.232423] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.232425] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.232428] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.232456] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1039, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:24.232613] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164325962BC3-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301561, send_timestamp=1719306001931052) [2024-06-25 17:00:24.232683] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-0006164325962BC4-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301569, send_timestamp=1719306001931114) [2024-06-25 17:00:24.233683] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABF4538-0006166954EF5A8B-0-0] [lt=6] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b61c69470, extra_info="") [2024-06-25 17:00:24.233762] INFO [LIB] stat (utility.h:1197) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=10] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=65, this=0x2b3b61c733b0, extra_info="palf_id:1006") [2024-06-25 17:00:24.233770] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99167][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1006, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=18604) [2024-06-25 17:00:24.233792] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=7, this=0x2b3b61c735e0, extra_info="palf_id:1006") [2024-06-25 17:00:24.233957] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BC5-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b61c696a0, extra_info="palf_id:1006") [2024-06-25 17:00:24.233962] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BC5-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=273, this=0x2b3b61c698d0, extra_info="palf_id:1006") [2024-06-25 17:00:24.235080] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=8] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:24.236047] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=4713221, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=197, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:24.237240] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=408100, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=225, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:24.240800] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.243560] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=0] will sleep(sleep_us=9000, remain_us=29425327, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=9, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:24.245182] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [99399][T1004_LSArchive][T1004][YB420ABF4533-00061642492E6725-0-0] [lt=8] gc stale ls task succ [2024-06-25 17:00:24.250171] INFO [PALF] handle_next_submit_log_ (log_sliding_window.cpp:1091) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF79B8-0-0] [lt=3] [PALF STAT GROUP LOG INFO](palf_id=1007, self="10.191.69.51:2882", role="FOLLOWER", total_group_log_cnt=10, avg_log_batch_cnt=0, total_group_log_size=1211, avg_group_log_size=121) [2024-06-25 17:00:24.250180] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [95591][T1002_L0_G1][T1002][YB420ABF4538-0006166954CF79B8-0-0] [lt=8] [PALF STAT COMMITTED LOG SIZE](palf_id=1007, self="10.191.69.51:2882", committed size=1211) [2024-06-25 17:00:24.252612] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62747][T1_ReplaySrv4][T1][Y0-0000000000000000-0-0] [lt=8] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:24.262964] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647494, offset=60301158}, end_cursor:ObLogCursor{file_id=100, log_id=27647495, offset=60301399}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbfc050, len:4250, log_data_len:241, seq:27647494, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:24.263003] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABF4538-00061669548FCCBB-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301561, send_timestamp=1719306001961442) [2024-06-25 17:00:24.264054] INFO [STORAGE.TRANS] get_gts (ob_gts_source.cpp:268) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8017-0-0] [lt=6] push gts task success(*task={:""}) [2024-06-25 17:00:24.265387] INFO [STORAGE.TRANS] statistics_ (ob_trans_rpc.cpp:670) [99180][T1004_TeRec][T1003][YB420ABF4533-00061661620F8017-0-0] [lt=9] rpc statistics(total_trans_msg_count=2, total_batch_msg_count=0) [2024-06-25 17:00:24.265943] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=14] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647495, offset=60301399}, end_cursor:ObLogCursor{file_id=100, log_id=27647496, offset=60301648}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbf4050, len:4009, log_data_len:249, seq:27647495, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:24.269279] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.269286] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.269289] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.269291] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:24.269293] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:24.269498] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-000616432547F0BE-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301555, send_timestamp=1719306001967943) [2024-06-25 17:00:24.269802] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=7] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647496, offset=60301648}, end_cursor:ObLogCursor{file_id=100, log_id=27647497, offset=60301897}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fbf8050, len:3760, log_data_len:249, seq:27647496, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:24.269882] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=6] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647497, offset=60301897}, end_cursor:ObLogCursor{file_id=100, log_id=27647498, offset=60302146}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fc04050, len:3511, log_data_len:249, seq:27647497, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:24.269972] INFO [STORAGE.REDO] notify_flush (ob_storage_log_writer.cpp:552) [99258][T1004_OB_SLOG][T1004][Y0-0000000000000000-0-0] [lt=5] Successfully flush(log_item={start_cursor:ObLogCursor{file_id=100, log_id=27647498, offset=60302146}, end_cursor:ObLogCursor{file_id=100, log_id=27647499, offset=60302395}, is_inited:true, is_local:false, buf_size:8192, buf:0x2b3a2fc14050, len:3262, log_data_len:249, seq:27647498, flush_finish:false, flush_ret:0}) [2024-06-25 17:00:24.270093] INFO eloop_run (eloop.c:143) [62250][pnio2][T0][Y0-0000000000000000-0-0] [lt=9] PNIO [ratelimit] time: 1719306024270092, bytes: 0, bw: 0.000000 MB/s, add_ts: 1007617, add_bytes: 0 [2024-06-25 17:00:24.279888] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62234][pnio1][T1001][YB420ABF4533-00061661718F6565-0-0] [lt=10][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:24.286278] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=9448546, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=166, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:24.288098] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.292760] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.292769] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:24.292775] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.292781] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.292786] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.295629] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.295637] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.295641] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:24.295647] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.295652] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.296702] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1044) [100423][T1003_FreInfoRe][T1003][Y0-0000000000000000-0-0] [lt=9] success to update min reserved snapshot(reserved_snapshot=1719304212419817000, duration=1800, snapshot_gc_ts_=1719306012419817000) [2024-06-25 17:00:24.296713] INFO [STORAGE] try_update_reserved_snapshot (ob_tenant_freeze_info_mgr.cpp:1071) [100423][T1003_FreInfoRe][T1003][Y0-0000000000000000-0-0] [lt=9] update reserved snapshot finished(cost_ts=13, reserved_snapshot=1719304212419817000) [2024-06-25 17:00:24.299243] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=1] will sleep(sleep_us=32000, remain_us=27732192, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=32, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:24.299554] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.299564] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:24.299569] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:3000000}) [2024-06-25 17:00:24.299576] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.299580] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.301506] INFO [SHARE] get_all_snapshots (ob_snapshot_table_proxy.cpp:426) [99249][T1004_FreInfoRe][T1004][Y0-0000000000000000-0-0] [lt=15] get all snapshots(ret=0, snapshots.count()=3, snapshots=[{snapshot_type:1, snapshot_scn:{val:1717463693595282000, v:0}, schema_version:1717463690222192, tenant_id:1004, tablet_id:208717, comment:null}, {snapshot_type:1, snapshot_scn:{val:1719194468064666000, v:0}, schema_version:1719194439686752, tenant_id:1004, tablet_id:209217, comment:null}, {snapshot_type:1, snapshot_scn:{val:1719194468064666000, v:0}, schema_version:1719194439686752, tenant_id:1004, tablet_id:1152921504606854157, comment:null}]) [2024-06-25 17:00:24.305376] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.305385] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.305389] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:2000000}) [2024-06-25 17:00:24.305394] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.305400] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.306441] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=0] will sleep(sleep_us=10000, remain_us=29362446, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=10, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:24.310143] INFO [CLOG] get_replay_process (ob_replay_status.cpp:1073) [99267][T1004_ReplayPro][T1004][Y0-0000000000000000-0-0] [lt=7] replay status is not follower(min_unreplayed_lsn={lsn:1750156420456}, base_lsn={lsn:0}, this={ls_id_:{id:1005}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:1, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1716561398006517, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:11603, ret_code_:0}}, next_to_submit_lsn_:{lsn:1750156420456}, next_to_submit_scn_:{val:1716561397833071015, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713332543407963001, v:0}, iterator_:{iterator_impl:{buf_:0x2b66c950f000, next_round_pread_size:3678208, curr_read_pos:50242, curr_read_buf_start_pos:0, curr_read_buf_end_pos:50242, log_storage_:{IteratorStorage:{start_lsn:{lsn:1750156370214}, end_lsn:{lsn:1750156420456}, read_buf:{buf_len_:294912, buf_:0x2b66c950f000}, block_size:67104768, log_storage_:0x2b3b5a81d470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1716561397833071014, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:32195, scn_:{val:1716561397833071014, v:0}, data_checksum:3072636752, flag:1}}, init_mode_version:0, accumulate_checksum:2076199808, curr_entry_is_padding:0, padding_entry_size:32227, padding_entry_scn:{val:1716561397833071014, v:0}}}}}) [2024-06-25 17:00:24.310175] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [99267][T1004_ReplayPro][T1004][Y0-0000000000000000-0-0] [lt=32] get_replay_process success(id={id:1005}, replayed_log_size=1750156420456, unreplayed_log_size=0) [2024-06-25 17:00:24.310180] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [99267][T1004_ReplayPro][T1004][Y0-0000000000000000-0-0] [lt=4] get_replay_process success(id={id:1006}, replayed_log_size=9248068461998, unreplayed_log_size=0) [2024-06-25 17:00:24.310184] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [99267][T1004_ReplayPro][T1004][Y0-0000000000000000-0-0] [lt=3] get_replay_process success(id={id:1007}, replayed_log_size=5896353989123, unreplayed_log_size=0) [2024-06-25 17:00:24.310189] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [99267][T1004_ReplayPro][T1004][Y0-0000000000000000-0-0] [lt=4] get_replay_process success(id={id:1008}, replayed_log_size=5880977363908, unreplayed_log_size=0) [2024-06-25 17:00:24.310193] INFO [CLOG] get_replay_process (ob_replay_status.cpp:1073) [99267][T1004_ReplayPro][T1004][Y0-0000000000000000-0-0] [lt=2] replay status is not follower(min_unreplayed_lsn={lsn:0}, base_lsn={lsn:0}, this={ls_id_:{id:1039}, is_enabled_:true, is_submit_blocked_:false, role_:1, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:1, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:0, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1716928608957087, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:0, ret_code_:0}}, next_to_submit_lsn_:{lsn:0}, next_to_submit_scn_:{val:0, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1716928602493732000, v:0}, iterator_:{iterator_impl:{buf_:null, next_round_pread_size:3678208, curr_read_pos:0, curr_read_buf_start_pos:0, curr_read_buf_end_pos:0, log_storage_:{IteratorStorage:{start_lsn:{lsn:0}, end_lsn:{lsn:0}, read_buf:{buf_len_:0, buf_:null}, block_size:67104768, log_storage_:0x2bcde052b470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:18446744073709551615, v:3}, curr_entry:{LogEntryHeader:{magic:0, version:0, log_size:-1, scn_:{val:18446744073709551615, v:3}, data_checksum:0, flag:0}}, init_mode_version:0, accumulate_checksum:-1, curr_entry_is_padding:0, padding_entry_size:0, padding_entry_scn:{val:18446744073709551615, v:3}}}}}) [2024-06-25 17:00:24.310210] INFO [CLOG] operator() (ob_log_replay_service.cpp:1590) [99267][T1004_ReplayPro][T1004][Y0-0000000000000000-0-0] [lt=17] get_replay_process success(id={id:1039}, replayed_log_size=0, unreplayed_log_size=0) [2024-06-25 17:00:24.310222] INFO [CLOG] runTimerTask (ob_log_replay_service.cpp:159) [99267][T1004_ReplayPro][T1004][Y0-0000000000000000-0-0] [lt=3] dump tenant replay process(tenant_id=1004, unreplayed_log_size(MB)=0, estimate_time(second)=0, replayed_log_size(MB)=21720463, last_replayed_log_size(MB)=21720462, round_cost_time(second)=10, pending_replay_log_size(MB)=0) [2024-06-25 17:00:24.315171] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.315179] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4013] [2024-06-25 17:00:24.315183] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.315189] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.315193] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.323966] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABF4533-0006166171EF489A-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301557, send_timestamp=1719306002022409) [2024-06-25 17:00:24.327814] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABA45AB-000616C810AFBBAE-0-0] [lt=3] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b74f470, extra_info="") [2024-06-25 17:00:24.327909] INFO [LIB] stat (utility.h:1197) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=79, this=0x2b3b5b7593b0, extra_info="palf_id:1007") [2024-06-25 17:00:24.327918] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1007, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1211) [2024-06-25 17:00:24.327927] INFO [LIB] stat (utility.h:1214) [99168][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT IO TASK IN QUEUE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=8, this=0x2b3afcbffaf0, extra_info="") [2024-06-25 17:00:24.327941] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=9, this=0x2b3b5b7595e0, extra_info="palf_id:1007") [2024-06-25 17:00:24.328650] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BC7-0-0] [lt=7] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5b74f6a0, extra_info="palf_id:1007") [2024-06-25 17:00:24.328654] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BC7-0-0] [lt=4] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=860, this=0x2b3b5b74f8d0, extra_info="palf_id:1007") [2024-06-25 17:00:24.328663] INFO [CLOG] handle_submit_task_ (ob_log_replay_service.cpp:1208) [99517][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=9] succ to submit log task to replay service(to_submit_lsn={lsn:5896353989179}, to_submit_scn={val:1719306024049345002, v:0}, replay_status={ls_id_:{id:1007}, is_enabled_:true, is_submit_blocked_:false, role_:2, err_info_:{lsn_:{lsn:18446744073709551615}, scn_:{val:0, v:0}, log_type_:0, is_submit_err_:false, err_ts_:0, err_ret_:0}, ref_cnt_:2, post_barrier_lsn_:{lsn:18446744073709551615}, pending_task_count_:1, submit_log_task_:{ObReplayServiceSubmitTask:{type_:1, enqueue_ts_:1719306024328648, err_info_:{has_fatal_error_:false, fail_ts_:0, fail_cost_:237788, ret_code_:0}}, next_to_submit_lsn_:{lsn:5896353989245}, next_to_submit_scn_:{val:1719306024049345003, v:0}, base_lsn_:{lsn:0}, base_scn_:{val:1713332543410073001, v:0}, iterator_:{iterator_impl:{buf_:0x2b4f5c205000, next_round_pread_size:3678208, curr_read_pos:122, curr_read_buf_start_pos:0, curr_read_buf_end_pos:122, log_storage_:{IteratorStorage:{start_lsn:{lsn:5896353989123}, end_lsn:{lsn:5896353989245}, read_buf:{buf_len_:3682304, buf_:0x2b4f5c205000}, block_size:67104768, log_storage_:0x2b3b5b755470}, IteratorStorageType::"DiskIteratorStorage"}, curr_entry_is_raw_write:false, curr_entry_size:0, prev_entry_scn:{val:1719306024049345002, v:0}, curr_entry:{LogEntryHeader:{magic:19528, version:1, log_size:34, scn_:{val:1719306024049345002, v:0}, data_checksum:2442502576, flag:0}}, init_mode_version:0, accumulate_checksum:53703714, curr_entry_is_padding:0, padding_entry_size:66, padding_entry_scn:{val:1719306024049345002, v:0}}}}}) [2024-06-25 17:00:24.331699] INFO [SQL.EXE] end_stmt (ob_sql_trans_control.cpp:1120) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] end stmt(ret=0, plain_select=true, stmt_type=1, savepoint=0, tx_desc={this:0x2b399e691700, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1002, session_id:3221516821, assoc_session_id:3221516821, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:1, alloc_ts:1719306024062644, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}, trans_result={incomplete:false, parts:[], touched_ls_list:[], cflict_txs:[]}, rollback=true, session={this:0x2b3aee2161f8, id:3221516821, deser:true, tenant:"", tenant_id:1002, effective_tenant:"", effective_tenant_id:1002, database:"ocp_express", user:"meta@%", consistency_level:3, session_state:2, autocommit:true, tx:0x2b399e691700}, exec_ctx.get_errcode()=0) [2024-06-25 17:00:24.332241] INFO [CLOG] get_max_applied_scn (ob_log_apply_service.cpp:732) [62719][T1_TenantWeakRe][T1][Y0-0000000000000000-0-0] [lt=21] get_max_applied_scn(scn={val:1, v:0}, this={ls_id_:{id:1}, role_:2, proposal_id_:-1, palf_committed_end_lsn_:{lsn:0}, last_check_scn_:{val:18446744073709551615, v:3}, max_applied_cb_scn_:{val:1, v:0}}) [2024-06-25 17:00:24.332435] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:24.332463] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:24.332469] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=5] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.332472] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.332476] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.332549] INFO [PALF] try_advance_committed_lsn_ (log_sliding_window.cpp:1748) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT COMMITTED LOG SIZE](palf_id=1005, self="10.191.69.51:2882", committed size=1220) [2024-06-25 17:00:24.332668] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62240][pnio1][T0][YB420ABDC060-0006164325962BC8-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301565, send_timestamp=1719306002031102) [2024-06-25 17:00:24.332668] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-0006164325962BC9-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301559, send_timestamp=1719306002031108) [2024-06-25 17:00:24.333052] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.333062] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:24.333066] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.333073] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.333078] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.335132] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:24.340855] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=8][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.342703] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=8] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=8) [2024-06-25 17:00:24.343866] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62244][pnio1][T0][YB420ABF4533-0006166172BEBEF4-0-0] [lt=11][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301567, send_timestamp=1719306002042298) [2024-06-25 17:00:24.343960] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [63467][T1001_ReqMemEvi][T1001][Y0-0000000000000000-0-0] [lt=11] sql audit evict task end(request_manager_->get_tenant_id()=1001, evict_high_mem_level=16106127, evict_high_size_level=90000, evict_batch_count=0, elapse_time=0, size_used=7444, mem_used=12478464) [2024-06-25 17:00:24.349879] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.349888] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=8][errcode=-4013] [2024-06-25 17:00:24.349892] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.349898] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.349904] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.362538] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62458][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.362769] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325C301A3-0-0] [lt=13][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301561, send_timestamp=1719306002061208) [2024-06-25 17:00:24.362859] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62460][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.363056] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABF4538-00061669548FCCBF-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301553, send_timestamp=1719306002061503) [2024-06-25 17:00:24.363100] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=0] will sleep(sleep_us=95000, remain_us=20364075, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=95, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:24.363138] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62457][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.363352] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62456][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.363421] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62454][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.363610] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62455][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.363630] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62459][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.364952] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABA45B0-00061645062C1BF5-0-0] [lt=4] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=0, this=0x2b3aaf09d470, extra_info="") [2024-06-25 17:00:24.364999] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [62725][T1_LSArchiveMgr][T1][YB420ABF4533-000616423A1E6725-0-0] [lt=7] gc stale ls task succ [2024-06-25 17:00:24.365048] INFO [LIB] stat (utility.h:1197) [63171][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=11] [PALF STAT WRITE LOG COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=76, this=0x2b3aaf0a73b0, extra_info="palf_id:1009") [2024-06-25 17:00:24.365070] INFO [LIB] stat (utility.h:1197) [63169][T1002_LogIOCb0][T1002][Y0-0000000000000000-0-0] [lt=6] [PALF STAT FLUSH CB COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=6, this=0x2b3aaf0a75e0, extra_info="palf_id:1009") [2024-06-25 17:00:24.365820] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0C1-0-0] [lt=9] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=2, this=0x2b3aaf09d6a0, extra_info="palf_id:1009") [2024-06-25 17:00:24.365825] INFO [LIB] stat (utility.h:1197) [95591][T1002_L0_G1][T1002][YB420ABDC060-000616432547F0C1-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=9, stat_interval=1000000, avg (count/cost)=848, this=0x2b3aaf09d8d0, extra_info="palf_id:1009") [2024-06-25 17:00:24.366351] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=17458627, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=114, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:24.366527] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62461][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.366743] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.366753] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:24.366758] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.366764] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.366769] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.366825] INFO [SERVER.OMT] check_worker_count (ob_tenant.cpp:488) [62538][MultiTenant][T0][Y0-0000000000000000-0-0] [lt=356] worker thread began to shrink(tenant_->id()=1002, group_id_=19, token=0) [2024-06-25 17:00:24.367276] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62467][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=5] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367317] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62469][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367328] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62464][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367350] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62471][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367362] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62435][BatchIO][T0][Y0-0000000000000000-0-0] [lt=9] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367405] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62466][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367572] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62468][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367708] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62470][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367910] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62465][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367928] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62445][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367934] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62444][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367937] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62462][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.367942] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62436][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368053] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62443][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368124] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62450][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368124] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62441][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368148] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62473][MysqlUnix][T0][Y0-0000000000000000-0-0] [lt=6] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368151] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62440][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368151] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62451][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368196] INFO [RPC.FRAME] mysql_easy_timer_cb (ob_net_easy.cpp:676) [62463][MysqlIO][T0][Y0-0000000000000000-0-0] [lt=7] [MYSQL EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368246] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62439][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368248] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62449][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368249] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62452][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368295] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62442][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368374] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62434][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368426] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62446][BatchIO][T0][Y0-0000000000000000-0-0] [lt=7] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368486] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62447][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368510] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62448][BatchIO][T0][Y0-0000000000000000-0-0] [lt=5] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368511] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62438][BatchIO][T0][Y0-0000000000000000-0-0] [lt=8] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368651] INFO [RPC.FRAME] batch_rpc_easy_timer_cb (ob_net_easy.cpp:641) [62437][BatchIO][T0][Y0-0000000000000000-0-0] [lt=6] [BATCH_RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.368877] INFO [SERVER.OMT] check_worker_count (ob_tenant.cpp:503) [95444][T1002_L0_G19][T1002][Y0-0000000000000000-0-0] [lt=13] worker thread exit(tenant_->id()=1002, workers_.get_size()=1) [2024-06-25 17:00:24.368885] INFO [SHARE] end_run (ob_tenant_base.cpp:332) [95444][T1002_L0_G19][T0][Y0-0000000000000000-0-0] [lt=6] tenant thread end_run(id_=1002, ret=0, thread_count_=189) [2024-06-25 17:00:24.368897] INFO unregister_pm (ob_page_manager.cpp:50) [95444][T1002_L0_G19][T0][Y0-0000000000000000-0-0] [lt=5] unregister pm finish(&pm=0x2bd048e56d80, pm.get_tid()=95444) [2024-06-25 17:00:24.369333] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.369340] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.369342] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.369345] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:24.369348] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:24.370297] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=0] will sleep(sleep_us=11000, remain_us=29298591, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=11, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:24.370955] INFO [COMMON] print_io_status (ob_io_manager.cpp:1358) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS](tenant_id=500, io_status=group_id: 0, group_name: OTHER_GROUPS, mode: read, size: 8192.00, iops: 0.93, rt: 94.00) [2024-06-25 17:00:24.370967] INFO [COMMON] print_io_status (ob_io_manager.cpp:1384) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=9] [IO STATUS SYS](tenant_id=500, io_status=sys_group_name: INSPECT_BAD_BLOCK_IO, mode: read, size: 2097152.00, iops: 0.93, rt: 768.00) [2024-06-25 17:00:24.370972] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS CONFIG](tenant_id=500, ref_cnt=2, io_config={group_num_:0, memory_limit_:536870912, callback_thread_count_:8, unit_config_:{min_iops:10000, max_iops:50000, weight:10000}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10866720, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:24.371016] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=17] [IO STATUS SYS](tenant_id=1002, io_status=sys_group_name: SLOG_IO, mode: write, size: 4096.00, iops: 3.72, rt: 66.75) [2024-06-25 17:00:24.371022] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=5] [IO STATUS CONFIG](tenant_id=1002, ref_cnt=2, io_config={group_num_:0, memory_limit_:1073741824, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:1}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=10735904, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:24.371049] INFO [COMMON] print_io_status (ob_io_manager.cpp:1393) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=8] [IO STATUS SYS](tenant_id=1004, io_status=sys_group_name: SLOG_IO, mode: write, size: 4915.20, iops: 4.65, rt: 75.20) [2024-06-25 17:00:24.371054] INFO [COMMON] print_io_status (ob_io_manager.cpp:1408) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS CONFIG](tenant_id=1004, ref_cnt=2, io_config={group_num_:0, memory_limit_:724775731200, callback_thread_count_:8, unit_config_:{min_iops:9223372036854775807, max_iops:9223372036854775807, weight:100}, enable_io_tracer:false, group_configs:[other_groups:{deleted:false, cleared:false, min_percent:100, max_percent:100, weight_percent:100}]}, allocated_memory=12763552, pre_allocated_count=2, callback_queues=[0, 0, 0, 0, 0, 0, 0, 0]) [2024-06-25 17:00:24.371063] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=7] [IO STATUS SENDER](send_index=1, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371070] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=6] [IO STATUS SENDER](send_index=2, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371073] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=3, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371076] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=4, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371081] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=5, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371084] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=6, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371087] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=7, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371089] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=8, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371093] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=9, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371096] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=10, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371100] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=11, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371103] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=12, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371106] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=2] [IO STATUS SENDER](send_index=13, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371110] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=4] [IO STATUS SENDER](send_index=14, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371113] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=15, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.371116] INFO [COMMON] print_sender_status (ob_io_struct.cpp:847) [62169][IO_TUNING0][T0][Y0-0000000000000000-0-0] [lt=3] [IO STATUS SENDER](send_index=16, req_count=0, reservation_ts=9223372036854775807, group_limitation_ts=9223372036854775807, tenant_limitation_ts=9223372036854775807, proportion_ts=9223372036854775807) [2024-06-25 17:00:24.374051] INFO [RPC.FRAME] rpc_easy_timer_cb (ob_net_easy.cpp:603) [62475][RpcUnix][T0][Y0-0000000000000000-0-0] [lt=6] [RPC EASY STAT](log_str=conn count=0/0, request done=0/0, request doing=0/0) [2024-06-25 17:00:24.377031] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100324][T1004_L0_G0][T1004][YB420ABF4539-000616464A7A8BBB-0-0] [lt=9][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:24.379938] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62236][pnio1][T1001][YB420ABF4533-00061661718F6566-0-0] [lt=8][errcode=-4013] tenant weak read service cluster heartbeat RPC fail(ret=-4013, rcode={code:-4013, msg:"", warnings:[]}, tenant_id_=1001, dst="10.191.69.52:2882", cluster_service_tablet_id={id:226}) [2024-06-25 17:00:24.380829] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [62735][T1_HAService][T1][Y0-0000000000000000-0-0] [lt=7] start do ls ha handler(ls_id_array_=[{id:1}]) [2024-06-25 17:00:24.381478] WDIAG [SERVER] inner_get_next_row (ob_all_virtual_sys_stat.cpp:288) [100160][T1004_L0_G0][T1004][YB420ABF4539-000616464A7A8BBB-0-0] [lt=10][errcode=-4008] execute fail(ret=-4008) [2024-06-25 17:00:24.384051] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62232][pnio1][T0][YB420ABF4533-0006166172BEBEF9-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5387, fly_ts=22301559, send_timestamp=1719306002082492) [2024-06-25 17:00:24.384893] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0] will sleep(sleep_us=33000, remain_us=27646543, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=33, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:24.386358] INFO [SHARE] get_next_sess_id (ob_active_session_guard.cpp:360) [95813][][T0][Y0-0000000000000000-0-0] [lt=0] succ to generate background session id(sessid=4449405730029568) [2024-06-25 17:00:24.386374] INFO register_pm (ob_page_manager.cpp:40) [95813][][T0][Y0-0000000000000000-0-0] [lt=11] register pm finish(ret=0, &pm=0x2b3c170d4d80, pm.get_tid()=95813, tenant_id=500) [2024-06-25 17:00:24.386397] INFO [SHARE] pre_run (ob_tenant_base.cpp:314) [95813][][T1002][Y0-0000000000000000-0-0] [lt=6] tenant thread pre_run(MTL_ID()=1002, ret=0, thread_count_=190) [2024-06-25 17:00:24.386397] INFO [SERVER.OMT] recv_group_request (ob_tenant.cpp:1292) [62245][pnio1][T0][YB420ABF4538-00061669439FB1F1-0-0] [lt=9] worker thread created(id()=1002, group->group_id_=19) [2024-06-25 17:00:24.386403] INFO [RPC.OBRPC] th_init (ob_rpc_translator.cpp:33) [95813][][T1002][Y0-0000000000000000-0-0] [lt=5] Init thread local success [2024-06-25 17:00:24.386409] INFO unregister_pm (ob_page_manager.cpp:50) [95813][][T1002][Y0-0000000000000000-0-0] [lt=4] unregister pm finish(&pm=0x2b3c170d4d80, pm.get_tid()=95813) [2024-06-25 17:00:24.386415] INFO register_pm (ob_page_manager.cpp:40) [95813][][T1002][Y0-0000000000000000-0-0] [lt=3] register pm finish(ret=0, &pm=0x2b3c170d4d80, pm.get_tid()=95813, tenant_id=1002) [2024-06-25 17:00:24.386826] INFO [COMMON] process (ob_rpc_processor_simple.cpp:2405) [95813][T1002_L0_G19][T1002][YB420ABF4538-00061669439FB1F1-0-0] [lt=4] handle ObQueryLSIsValidMember requeset(arg_={tenant_id_:1002, self_addr_:"10.191.69.56:2882", ls_array_:[{id:1006}]}) [2024-06-25 17:00:24.386837] INFO [CLOG] get_member_gc_stat (ob_log_handler.cpp:1509) [95813][T1002_L0_G19][T1002][YB420ABF4538-00061669439FB1F1-0-0] [lt=9] get_member_gc_stat(is_valid_member=true, stat=1, member_list=5{server:"10.186.69.171:2882", timestamp:1, flag:0}{server:"10.186.69.176:2882", timestamp:1, flag:0}{server:"10.189.192.96:2882", timestamp:1, flag:0}{server:"10.191.69.51:2882", timestamp:1, flag:0}{server:"10.191.69.56:2882", timestamp:1, flag:0}, learner_list={learner_num:0, learner_array:[]}, addr="10.191.69.56:2882", this={role:1, proposal_id:1, palf_env_:0x2b3a8ce16030, is_in_stop_state_:false, is_inited_:true, id_:1006}) [2024-06-25 17:00:24.388156] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.389793] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=0] will sleep(sleep_us=100000, remain_us=4559475, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=198, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:24.390778] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=1] will sleep(sleep_us=100000, remain_us=254563, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=226, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:24.421868] INFO [CLOG] run1 (ob_garbage_collector.cpp:1338) [62697][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=3] Garbage Collector is running(seq_=598190, gc_interval=10000000) [2024-06-25 17:00:24.422703] INFO [CLOG] handle_rpc_response_ (ob_garbage_collector.cpp:272) [62697][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=10] GC check ls in member list, skip it(id={id:1}, leader="10.186.69.171:2882") [2024-06-25 17:00:24.422712] INFO [CLOG] gc_check_member_list_ (ob_garbage_collector.cpp:1431) [62697][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=7] gc_check_member_list_ cost time(ret=0, time_us=834) [2024-06-25 17:00:24.422717] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1703) [62697][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=4] execute_gc cost time(ret=0, time_us=0) [2024-06-25 17:00:24.422723] INFO [CLOG] gc_check_ls_status_ (ob_garbage_collector.cpp:1626) [62697][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=4] gc_candidates push_back success(ret=0, candidate={ls_id_:{id:1}, ls_status_:1, gc_reason_:0}) [2024-06-25 17:00:24.422731] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1648) [62697][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=6] ls status is normal, skip(id={id:1}, gc_candidates=[{ls_id_:{id:1}, ls_status_:1, gc_reason_:0}]) [2024-06-25 17:00:24.422735] INFO [CLOG] execute_gc_ (ob_garbage_collector.cpp:1703) [62697][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=4] execute_gc cost time(ret=0, time_us=5) [2024-06-25 17:00:24.422738] INFO [SERVER] handle (ob_safe_destroy_handler.cpp:240) [62697][T1_GCCollector][T1][Y0-0000000000000000-0-0] [lt=3] ObSafeDestroyHandler start process [2024-06-25 17:00:24.425844] INFO [COMMON] compute_tenant_wash_size (ob_kvcache_store.cpp:1157) [62192][KVCacheWash][T0][Y0-0000000000000000-0-0] [lt=12] Wash compute wash size(is_wash_valid=false, sys_total_wash_size=-289089040384, global_cache_size=552067284992, tenant_max_wash_size=79782001, tenant_min_wash_size=79782001, tenant_ids_=[500, 508, 1, 1001, 1002, 1003, 1004]) [2024-06-25 17:00:24.428773] INFO [LIB] stat (utility.h:1197) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=10] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=8, this=0x2b3ae6e6c8f8, extra_info="") [2024-06-25 17:00:24.428784] INFO [LIB] stat (utility.h:1197) [99520][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=11] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=30, stat_interval=1000000, avg (count/cost)=10, this=0x2b3ae6e6cb28, extra_info="") [2024-06-25 17:00:24.429147] INFO [SQL.QRR] runTimerTask (ob_udr_mgr.cpp:92) [100428][T1003_ReqMemEvi][T1003][Y0-0000000000000000-0-0] [lt=8] run rewrite rule refresh task(rule_mgr_->tenant_id_=1003) [2024-06-25 17:00:24.432495] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:24.432530] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:24.432534] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.432537] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.432541] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.432688] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.432698] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=9][errcode=-4013] [2024-06-25 17:00:24.432704] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:1000000}) [2024-06-25 17:00:24.432711] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.432716] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.432720] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62248][pnio1][T0][YB420ABDC060-0006164325962BCE-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301552, send_timestamp=1719306002131167) [2024-06-25 17:00:24.432722] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62247][pnio1][T0][YB420ABDC060-0006164325962BCD-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301561, send_timestamp=1719306002131160) [2024-06-25 17:00:24.433985] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BCF-0-0] [lt=3] [PALF STAT LOG TASK TIME](palf_id=1006, self="10.191.69.51:2882", total_slide_log_cnt=11, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=2, avg_log_submit_to_flush_time=79, avg_log_submit_to_slide_time=273) [2024-06-25 17:00:24.433994] INFO [PALF] read (log_cache.cpp:100) [99266][T1004_ReplaySrv][T1004][Y0-0000000000000000-0-0] [lt=12] [PALF STAT HOT CACHE HIT RATE](palf_id=1006, read_size=18613, hit_cnt=11, read_cnt=11, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:24.434306] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62233][pnio1][T0][YB420ABDC060-0006164325210E45-0-0] [lt=7][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=330, fly_ts=22301553, send_timestamp=1719306002132752) [2024-06-25 17:00:24.434764] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8042FDE57-0-0] [lt=4] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1006, child={server:"10.186.69.171:2882", region:"sys_region", register_time_us:1719306024433897, keepalive_ts:-1}) [2024-06-25 17:00:24.434776] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95591][T1002_L0_G1][T1002][YB420ABA45AB-000616C8042FDE57-0-0] [lt=12] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1006, self="10.191.69.51:2882", child={server:"10.186.69.171:2882", region:"sys_region", register_time_us:1719306024433897, keepalive_ts:-1}, is_to_leader=true) [2024-06-25 17:00:24.435151] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=0] will sleep(sleep_us=12000, remain_us=29233736, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=12, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:24.435190] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:24.436928] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [62168][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=9][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4383, dropped:957, tid:63185}]) [2024-06-25 17:00:24.437513] INFO [STORAGE] gc_tables_in_queue (ob_tenant_meta_mem_mgr.cpp:497) [63257][T1002_TenantMet][T1002][Y0-0000000000000000-0-0] [lt=20] Recycle 0 table(ret=0, tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl3916EEE", sizeof(T):3952, used_obj_cnt:354, free_obj_hold_cnt:479, allocator used:3312008, allocator total:3466624}, large_tablet_buffer_pool_={typeid(T).name():"N9oceanbase7storage15ObMetaObjBufferINS0_8ObTabletELl65448EEE", sizeof(T):65480, used_obj_cnt:395, free_obj_hold_cnt:5, allocator used:26236400, allocator total:26236400}, full_tablet_creator_={tiny_allocator_.used():0, tiny_allocator_.total():123008, full allocator used:1600, full allocator total:188544}, tablets_mem=29891568, tablets_mem_limit=214748360, ddl_kv_pool_={typeid(T).name():"N9oceanbase7storage7ObDDLKVE", sizeof(T):2368, used_obj_cnt:0, free_obj_hold_cnt:0, allocator used:0, allocator total:0}, memtable_pool_={typeid(T).name():"N9oceanbase8memtable10ObMemtableE", sizeof(T):1920, used_obj_cnt:2, free_obj_hold_cnt:732, allocator used:1456256, allocator total:1504384}, pending_cnt=0, wait_gc_count=0, tablet count=749) [2024-06-25 17:00:24.437749] INFO [PALF] sliding_cb (log_sliding_window.cpp:2261) [95515][T1_L0_G1][T1][YB420ABDC060-0006164324F3A0BD-0-0] [lt=9] [PALF STAT LOG TASK TIME](palf_id=1, self="10.191.69.51:2882", total_slide_log_cnt=12, avg_log_gen_to_freeze_time=0, avg_log_gen_to_submit_time=3, avg_log_submit_to_flush_time=96, avg_log_submit_to_slide_time=845) [2024-06-25 17:00:24.437756] INFO [PALF] read (log_cache.cpp:100) [62745][T1_ReplaySrv2][T1][Y0-0000000000000000-0-0] [lt=12] [PALF STAT HOT CACHE HIT RATE](palf_id=1, read_size=4692, hit_cnt=12, read_cnt=12, hit rate=1.000000000000000000e+00) [2024-06-25 17:00:24.437829] INFO [COMMON] advance_clock (ob_fifo_arena.cpp:374) [62747][T1_ReplaySrv4][T1][Y0-0000000000000000-0-0] [lt=8] current clock is (clock_=0, max_seq_=0, mem_limit=11834648158, cur_mem_hold=285212672, attr_.tenant_id_=1) [2024-06-25 17:00:24.440156] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:24.440163] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=7][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:24.440167] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6934a400, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024385941, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.440186] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=19][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6934a400, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024385941, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.440197] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:24.440204] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=6][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:24.440207] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:24.440210] WDIAG [SQL] open (ob_result_set.cpp:157) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:24.440213] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:24.440216] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:24.440222] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=167, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:24.440228] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=9294596, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=167, v.err_=-4383, timeout_timestamp=1719306033734823) [2024-06-25 17:00:24.440910] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.442586] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62239][pnio1][T0][YB420ABDC060-0006164245661A9E-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1282, fly_ts=22301567, send_timestamp=1719306002141019) [2024-06-25 17:00:24.443314] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] execute rpc fail(addr="10.191.69.52:2882", pcode=1319, ret=-4013) [2024-06-25 17:00:24.443325] WDIAG log_user_error_and_warn (ob_poc_rpc_proxy.cpp:229) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=10][errcode=-4013] [2024-06-25 17:00:24.443330] WDIAG [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:112) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=3][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", server="10.191.69.52:2882", msg={tenant_id:1002, range:4000000}) [2024-06-25 17:00:24.443336] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.443342] WDIAG [SQL.DAS] get_das_id (ob_das_id_cache.cpp:166) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=6][errcode=-4013] fetch new range failed(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED") [2024-06-25 17:00:24.446171] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.447221] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=10][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:12, local_retry_times:12, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:24.447235] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=13][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.447239] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.447243] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:24.447248] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:24.449974] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:2000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.455868] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.457573] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABA45B0-00061645064ABF50-0-0] [lt=7] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5b6d5470, extra_info="") [2024-06-25 17:00:24.457667] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=76, this=0x2b3b5b6df3b0, extra_info="palf_id:1008") [2024-06-25 17:00:24.457692] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=6] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=9, this=0x2b3b5b6df5e0, extra_info="palf_id:1008") [2024-06-25 17:00:24.458166] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=7][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:95, local_retry_times:95, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:24.458176] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.458179] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.458181] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:24.458185] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:24.458440] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BD0-0-0] [lt=8] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b5b6d56a0, extra_info="palf_id:1008") [2024-06-25 17:00:24.458445] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BD0-0-0] [lt=5] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=849, this=0x2b3b5b6d58d0, extra_info="palf_id:1008") [2024-06-25 17:00:24.462547] INFO [STORAGE] runTimerTask (ob_checkpoint_service.cpp:346) [63471][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=4] ====== check clog disk timer task ====== [2024-06-25 17:00:24.462557] INFO [PALF] get_disk_usage (palf_env_impl.cpp:890) [63471][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=8] get_disk_usage(ret=0, capacity(MB):=614, used(MB):=0) [2024-06-25 17:00:24.462565] INFO [STORAGE] cannot_recycle_log_over_threshold_ (ob_checkpoint_service.cpp:260) [63471][T1001_CKClogDis][T1001][Y0-0000000000000000-0-0] [lt=4] cannot_recycle_log_size statistics(cannot_recycle_log_size=0, threshold=193273528, need_update_checkpoint_scn=false) [2024-06-25 17:00:24.462815] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62246][pnio1][T0][YB420ABDC060-0006164325C301A7-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301561, send_timestamp=1719306002161254) [2024-06-25 17:00:24.465655] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.466258] INFO [STORAGE.TRANS] operator() (ob_ts_mgr.h:177) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=12] refresh gts functor(ret=0, ret="OB_SUCCESS", gts_tenant_info={v:1}) [2024-06-25 17:00:24.466283] INFO [STORAGE.TRANS] refresh_gts (ob_gts_source.cpp:515) [62414][TsMgr][T0][Y0-0000000000000000-0-0] [lt=7] refresh gts(ret=0, ret="OB_SUCCESS", tenant_id=1003, need_refresh=false, gts_local_cache={srr:[mts=1719306024267967], gts:1719306024268042000, latest_srr:[mts=1719306024466278]}) [2024-06-25 17:00:24.466417] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=8][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:114, local_retry_times:114, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:24.466431] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=12][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.466435] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.466439] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:24.466446] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:24.466533] WDIAG [STORAGE.TRANS] process_ (ob_gts_rpc.h:196) [62243][pnio1][T0][YB420ABF4533-0006166171AF660B-0-0] [lt=8][errcode=0] gts rpc error(rcode={code:-4013, msg:"", warnings:[]}, dst="10.191.69.52:2882") [2024-06-25 17:00:24.466541] INFO [STORAGE.TRANS] process_ (ob_gts_rpc.h:198) [62243][pnio1][T0][YB420ABF4533-0006166171AF660B-0-0] [lt=7] get gts need refresh gts location(status=-4013, result={tenant_id:0, status:0, srr:[mts=0], gts_start:0, gts_end:0}) [2024-06-25 17:00:24.466553] INFO [STORAGE.TRANS] refresh_gts_location_ (ob_gts_source.cpp:579) [62243][pnio1][T0][YB420ABF4533-0006166171AF660B-0-0] [lt=5] gts nonblock renew success(ret=0, tenant_id=1001, gts_local_cache={srr:[mts=1718277123021980], gts:1718277123020766000, latest_srr:[mts=1719306024466268]}) [2024-06-25 17:00:24.467338] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.469393] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.469401] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=7] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.469404] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.469406] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:24.469410] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:24.469635] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62235][pnio1][T0][YB420ABDC060-000616432547F0C6-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301560, send_timestamp=1719306002168075) [2024-06-25 17:00:24.470211] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.471743] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=0][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:24.471754] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:24.471758] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb2188660, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024417384, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.471783] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=24][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb2188660, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024417384, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.471794] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999990, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:24.471803] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=6][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:24.471807] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:24.471811] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:24.471815] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:24.471822] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=6][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:24.471826] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=34, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:24.471833] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=6] will sleep(sleep_us=34000, remain_us=27559602, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=34, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:24.474110] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.475563] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABA45B0-00061644F03C36DF-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1008}}) [2024-06-25 17:00:24.475581] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABA45B0-00061644F03C36DF-0-0] [lt=9] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306024143939003, v:0}, arg={tenant_id:1004, ls_id:{id:1008}}, result={tenant_id:1004, ls_id:{id:1008}, cur_readable_scn:{val:1719306024143939003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:24.478170] WDIAG [SHARE] inner_get_ls_status_ (ob_ls_status_operator.cpp:955) [62688][T1_DupTbLease][T1][Y0-0000000000000000-0-0] [lt=8][errcode=-4018] ls not exist in __all_ls_status table(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-06-25 17:00:24.478201] WDIAG [SHARE] get_duplicate_ls_status_info (ob_ls_status_operator.cpp:674) [62688][T1_DupTbLease][T1][Y0-0000000000000000-0-0] [lt=10][errcode=-4018] fail to inner get ls status info(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sql=SELECT * FROM __all_ls_status where tenant_id = 1 and flag like "%DUPLICATE%", tenant_id=1, exec_tenant_id=1, need_member_list=false) [2024-06-25 17:00:24.479920] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.479998] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62247][pnio1][T1001][YB420ABF4533-00061661718F6567-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:24.480852] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3037) [100566][T1003_L0_G0][T1003][YB420ABA45AB-000616C80A1FC7FA-0-0] [lt=11] start get_ls_replayed_scn(arg={tenant_id:1004, ls_id:{id:1007}}) [2024-06-25 17:00:24.480861] INFO [SERVER] get_ls_replayed_scn (ob_service.cpp:3074) [100566][T1003_L0_G0][T1004][YB420ABA45AB-000616C80A1FC7FA-0-0] [lt=5] finish get_ls_replayed_scn(ret=0, ret="OB_SUCCESS", cur_readable_scn={val:1719306024049345003, v:0}, arg={tenant_id:1004, ls_id:{id:1007}}, result={tenant_id:1004, ls_id:{id:1007}, cur_readable_scn:{val:1719306024049345003, v:0}, offline_scn:{val:18446744073709551615, v:3}}, offline_scn={val:18446744073709551615, v:3}) [2024-06-25 17:00:24.483492] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=1] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.486887] INFO [ARCHIVE] gc_stale_ls_task_ (ob_ls_mgr.cpp:539) [100471][T1003_LSArchive][T1003][YB420ABF4533-0006164249CE6725-0-0] [lt=6] gc stale ls task succ [2024-06-25 17:00:24.488225] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [63291][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=7][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.489727] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.489855] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=10][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:198, local_retry_times:198, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:24.489868] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=12][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.489872] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.489875] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:24.489882] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:24.490845] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=9][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:226, local_retry_times:226, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:24.490855] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.490858] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.490861] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:24.490867] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:24.493813] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [100481][T1003_HAService][T1003][Y0-0000000000000000-0-0] [lt=10] start do ls ha handler(ls_id_array_=[]) [2024-06-25 17:00:24.500083] INFO [STORAGE] scheduler_ls_ha_handler_ (ob_storage_ha_service.cpp:189) [99409][T1004_HAService][T1004][Y0-0000000000000000-0-0] [lt=7] start do ls ha handler(ls_id_array_=[{id:1007}, {id:1008}, {id:1005}, {id:1006}, {id:1039}]) [2024-06-25 17:00:24.501044] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:24.501053] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=8][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:24.501057] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78dcd240, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024446831, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.501074] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78dcd240, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024446831, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.501085] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:24.501093] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=6][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:24.501096] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=2][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:24.501099] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:24.501102] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:24.501106] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:24.501111] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=13, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:24.501117] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=5] will sleep(sleep_us=13000, remain_us=29167771, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=13, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:24.505895] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=7][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:34, local_retry_times:34, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:24.505906] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=10][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.505910] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.505912] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:24.505918] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:24.507609] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.511088] INFO [STORAGE.BLKMGR] inspect_bad_block (ob_block_manager.cpp:1712) [62325][BlkMgr][T0][Y0-0000000000000000-0-0] [lt=12] check macro block(block_info={ref_cnt:1, is_free:false, access_time:1718072273190083}, time_interval=1233751321003) [2024-06-25 17:00:24.511988] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=4][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:24.511998] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:24.512002] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6d6b7e20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024457352, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.512020] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=18][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6d6b7e20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024457352, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.512031] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999993, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:24.512038] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:24.512041] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:24.512044] WDIAG [SQL] open (ob_result_set.cpp:157) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:24.512048] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:24.512050] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:24.512055] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=96, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:24.512061] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62274][TbltTblUp2][T1001][YB420ABF4533-000616423CAF16EE-0-0] [lt=5] will sleep(sleep_us=96000, remain_us=20215114, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=96, v.err_=-4383, timeout_timestamp=1719306044727175) [2024-06-25 17:00:24.514184] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=5][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:13, local_retry_times:13, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:24.514191] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=5][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.514193] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=2][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.514196] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:24.514202] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:24.517363] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.520115] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:24.520124] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:24.520129] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a678ea080, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024465766, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.520149] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=19][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a678ea080, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024465766, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.520161] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=11][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:24.520167] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:24.520172] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:24.520175] WDIAG [SQL] open (ob_result_set.cpp:157) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:24.520178] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:24.520184] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=5][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:24.520188] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=115, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:24.520195] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62273][TbltTblUp1][T1001][YB420ABF4533-000616423CCF159E-0-0] [lt=6] will sleep(sleep_us=100000, remain_us=17304784, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=115, v.err_=-4383, timeout_timestamp=1719306041824978) [2024-06-25 17:00:24.528378] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [63173][T1002_IOWorker][T1002][Y0-0000000000000000-0-0] [lt=4] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:24.532549] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1039}) [2024-06-25 17:00:24.532578] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=8] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=16, this=0x2b3b5a817470, extra_info="") [2024-06-25 17:00:24.532589] INFO [LIB] stat (utility.h:1197) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=7] [PALF STAT APPEND COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=0, this=0x2b3b5a884e50, extra_info="id:1005") [2024-06-25 17:00:24.532592] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1005}) [2024-06-25 17:00:24.532595] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.532597] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.532600] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [99410][T1004_TxLoopWor][T1004][Y0-0000000000000000-0-0] [lt=3] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.532666] INFO [LIB] stat (utility.h:1197) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT WRITE LOG COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=73, this=0x2b3b5a8213b0, extra_info="palf_id:1005") [2024-06-25 17:00:24.532685] INFO [LIB] stat (utility.h:1197) [99164][T1004_LogIOCb0][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT FLUSH CB COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=3, this=0x2b3b5a8215e0, extra_info="palf_id:1005") [2024-06-25 17:00:24.532771] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62242][pnio1][T0][YB420ABDC060-0006164325962BD2-0-0] [lt=10][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301564, send_timestamp=1719306002231206) [2024-06-25 17:00:24.532830] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62243][pnio1][T0][YB420ABDC060-0006164325962BD3-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301560, send_timestamp=1719306002231269) [2024-06-25 17:00:24.532866] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BD3-0-0] [lt=2] [PALF STAT FS CB EXCUTE COST TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=2, this=0x2b3b5a8176a0, extra_info="palf_id:1005") [2024-06-25 17:00:24.532873] INFO [LIB] stat (utility.h:1197) [95800][T1004_L0_G1][T1004][YB420ABDC060-0006164325962BD3-0-0] [lt=6] [PALF STAT LOG LIFE TIME](cur_stat_count=10, stat_interval=1000000, avg (count/cost)=297, this=0x2b3b5a8178d0, extra_info="palf_id:1005") [2024-06-25 17:00:24.533479] INFO [SHARE.LOCATION] dump_cache (ob_ls_location_service.cpp:1206) [62317][DumpLSLoc][T0][YB420ABF4533-00061660989F8BA7-0-0] [lt=142] [LS_LOCATION]dump tenant ls location caches(tenant_id=1, tenant_ls_locations=[{cache_key:{tenant_id:1, ls_id:{id:1}, cluster_id:1713323324}, renew_time:1719306023782334, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.186.69.176:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.96:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}]) [2024-06-25 17:00:24.533503] INFO [SHARE.LOCATION] dump_cache (ob_ls_location_service.cpp:1206) [62317][DumpLSLoc][T0][YB420ABF4533-00061660989F8BA7-0-0] [lt=23] [LS_LOCATION]dump tenant ls location caches(tenant_id=1001, tenant_ls_locations=[{cache_key:{tenant_id:1001, ls_id:{id:1}, cluster_id:1713323324}, renew_time:1719306024517002, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:10}, {server:"10.191.69.57:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}]) [2024-06-25 17:00:24.533529] INFO [SHARE.LOCATION] dump_cache (ob_ls_location_service.cpp:1206) [62317][DumpLSLoc][T0][YB420ABF4533-00061660989F8BA7-0-0] [lt=11] [LS_LOCATION]dump tenant ls location caches(tenant_id=1002, tenant_ls_locations=[{cache_key:{tenant_id:1002, ls_id:{id:1006}, cluster_id:1713323324}, renew_time:1718282374958161, replica_locations:[{server:"10.186.69.171:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.176:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.96:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1005}, cluster_id:1713323324}, renew_time:1718282374958159, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.97:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.191.69.52:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.57:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1001}, cluster_id:1713323324}, renew_time:1718282374958150, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.191.69.57:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1}, cluster_id:1713323324}, renew_time:1718282374958147, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.191.69.57:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1009}, cluster_id:1713323324}, renew_time:1718282374958168, replica_locations:[{server:"10.186.69.171:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.176:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.189.192.96:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1003}, cluster_id:1713323324}, renew_time:1718282374958155, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.57:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1008}, cluster_id:1713323324}, renew_time:1718282374958166, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.186.69.176:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.96:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1007}, cluster_id:1713323324}, renew_time:1718282374958164, replica_locations:[{server:"10.186.69.171:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.176:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.96:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}]}, {cache_key:{tenant_id:1002, ls_id:{id:1004}, cluster_id:1713323324}, renew_time:1718282374958157, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.57:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1010}, cluster_id:1713323324}, renew_time:1718282374958171, replica_locations:[{server:"10.186.69.171:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.176:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.96:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.191.69.51:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1002, ls_id:{id:1002}, cluster_id:1713323324}, renew_time:1718282374958152, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}]}]) [2024-06-25 17:00:24.533616] INFO [SHARE.LOCATION] dump_cache (ob_ls_location_service.cpp:1206) [62317][DumpLSLoc][T0][YB420ABF4533-00061660989F8BA7-0-0] [lt=85] [LS_LOCATION]dump tenant ls location caches(tenant_id=1003, tenant_ls_locations=[{cache_key:{tenant_id:1003, ls_id:{id:1}, cluster_id:1713323324}, renew_time:1719306023785756, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.191.69.57:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}]) [2024-06-25 17:00:24.533652] INFO [SHARE.LOCATION] dump_cache (ob_ls_location_service.cpp:1206) [62317][DumpLSLoc][T0][YB420ABF4533-00061660989F8BA7-0-0] [lt=11] [LS_LOCATION]dump tenant ls location caches(tenant_id=1004, tenant_ls_locations=[{cache_key:{tenant_id:1004, ls_id:{id:1003}, cluster_id:1713323324}, renew_time:1719306023787656, replica_locations:[{server:"10.186.69.173:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.57:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1030}, cluster_id:1713323324}, renew_time:1716856433384918, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1027}, cluster_id:1713323324}, renew_time:1716575755178528, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1007}, cluster_id:1713323324}, renew_time:1719306023787665, replica_locations:[{server:"10.186.69.171:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.186.69.176:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.96:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1020}, cluster_id:1713323324}, renew_time:1716561133327671, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1001}, cluster_id:1713323324}, renew_time:1719306023787652, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:3}, {server:"10.191.69.57:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1036}, cluster_id:1713323324}, renew_time:1716906886015906, replica_locations:[{server:"10.186.69.176:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1019}, cluster_id:1713323324}, renew_time:1716554264658094, replica_locations:[{server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"10.191.69.57:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1038}, cluster_id:1713323324}, renew_time:1716928611473983, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1010}, cluster_id:1713323324}, renew_time:1716465789656061, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1028}, cluster_id:1713323324}, renew_time:1716834822763991, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1004}, cluster_id:1713323324}, renew_time:1719306023787658, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.177:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.57:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1002}, cluster_id:1713323324}, renew_time:1719306023787654, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}]}, {cache_key:{tenant_id:1004, ls_id:{id:1012}, cluster_id:1713323324}, renew_time:1716465889855102, replica_locations:[{server:"10.186.69.171:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.96:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1026}, cluster_id:1713323324}, renew_time:1716575589859579, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1022}, cluster_id:1713323324}, renew_time:1716561708415976, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1006}, cluster_id:1713323324}, renew_time:1719306023787663, replica_locations:[{server:"10.186.69.171:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.176:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.96:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.56:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}]}, {cache_key:{tenant_id:1004, ls_id:{id:1011}, cluster_id:1713323324}, renew_time:1716465804678031, replica_locations:[{server:"10.191.69.57:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1039}, cluster_id:1713323324}, renew_time:1719306023787670, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.96:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:1}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1031}, cluster_id:1713323324}, renew_time:1716856528247427, replica_locations:[{server:"10.186.69.173:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1023}, cluster_id:1713323324}, renew_time:1716561718435550, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1024}, cluster_id:1713323324}, renew_time:1716568310805579, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:3}]}, {cache_key:{tenant_id:1004, ls_id:{id:1015}, cluster_id:1713323324}, renew_time:1716532535329240, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1005}, cluster_id:1713323324}, renew_time:1719306023787660, replica_locations:[{server:"10.186.69.171:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.176:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.189.192.96:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:3}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1016}, cluster_id:1713323324}, renew_time:1716546765694125, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1029}, cluster_id:1713323324}, renew_time:1716835279372264, replica_locations:[{server:"10.189.192.97:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1034}, cluster_id:1713323324}, renew_time:1716899692836780, replica_locations:[{server:"10.186.69.176:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}, {cache_key:{tenant_id:1004, ls_id:{id:1025}, cluster_id:1713323324}, renew_time:1716570888454805, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1014}, cluster_id:1713323324}, renew_time:1716532459466378, replica_locations:[{server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1037}, cluster_id:1713323324}, renew_time:1716907011234398, replica_locations:[{server:"10.186.69.177:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1035}, cluster_id:1713323324}, renew_time:1716899843124917, replica_locations:[{server:"10.191.69.52:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}]}, {cache_key:{tenant_id:1004, ls_id:{id:1032}, cluster_id:1713323324}, renew_time:1716878032255011, replica_locations:[{server:"10.186.69.171:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.186.69.176:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}, {server:"10.191.69.51:2882", role:1, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:2}, {server:"10.191.69.56:2882", role:2, sql_port:2881, replica_type:0, property:{memstore_percent_:100}, restore_status:{status:0}, proposal_id:0}]}]) [2024-06-25 17:00:24.535245] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [62736][T1_TxLoopWorker][T1][Y0-0000000000000000-0-0] [lt=9] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1}) [2024-06-25 17:00:24.536282] INFO [SERVER] runTimerTask (ob_eliminate_task.cpp:222) [100464][T1003_ReqMemEvi][T1003][Y0-0000000000000000-0-0] [lt=9] sql audit evict task end(request_manager_->get_tenant_id()=1003, evict_high_mem_level=1932735283, evict_high_size_level=90000, evict_batch_count=0, elapse_time=1, size_used=83052, mem_used=195495936) [2024-06-25 17:00:24.540291] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=7][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:167, local_retry_times:167, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:24.540301] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=9][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.540304] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.540306] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:24.540310] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62277][TbltTblUp5][T1001][YB420ABF4533-000616423D0F1736-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:24.540965] WDIAG [STORAGE.TRANS] run1 (ob_standby_timestamp_service.cpp:147) [99391][T1004_STSWorker][T1004][Y0-0000000000000000-0-0] [lt=9][errcode=-4076] query and update last id fail(ret=-4076, ret="OB_NEED_WAIT") [2024-06-25 17:00:24.541400] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [65259][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EF3-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:1000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.541635] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62237][pnio1][T0][YB420ABDC061-00061643952B83C1-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=1193, fly_ts=50620720, send_timestamp=1719305973920914) [2024-06-25 17:00:24.542767] INFO [COMMON] replace_map (ob_kv_storecache.cpp:746) [62193][KVCacheRep][T0][Y0-0000000000000000-0-0] [lt=6] replace map num details(ret=0, replace_node_count=0, map_once_replace_num_=100000, map_replace_skip_count_=7) [2024-06-25 17:00:24.543498] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:24.543508] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:24.543511] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b399e765100, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024488926, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.543530] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=18][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b399e765100, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024488926, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.543542] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999994, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:24.543548] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:24.543552] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:24.543556] WDIAG [SQL] open (ob_result_set.cpp:157) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=3][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:24.543559] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:24.543562] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:24.543567] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=5][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=199, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:24.543572] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62275][TbltTblUp3][T1001][YB420ABF4533-000616423CFF1809-0-0] [lt=4] will sleep(sleep_us=100000, remain_us=4405695, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=199, v.err_=-4383, timeout_timestamp=1719306028949267) [2024-06-25 17:00:24.544576] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=6][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:24.544586] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=9][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:24.544589] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a6ab31e20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024489979, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.544607] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a6ab31e20, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024489979, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.544619] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999998, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:24.544625] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:24.544629] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:24.544632] WDIAG [SQL] open (ob_result_set.cpp:157) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:24.544635] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:24.544639] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}) [2024-06-25 17:00:24.544643] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=3][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION WITH CONSISTENT SNAPSHOT"}, retry_cnt=227, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:24.544649] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62539][OmtNodeBalancer][T1001][YB420ABF4533-0006165E075FC741-0-0] [lt=5] will sleep(sleep_us=100000, remain_us=100691, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=227, v.err_=-4383, timeout_timestamp=1719306024645340) [2024-06-25 17:00:24.557698] INFO [PALF] inner_append_log (palf_handle_impl.cpp:2139) [99166][T1004_IOWorker][T1004][Y0-0000000000000000-0-0] [lt=9] [PALF STAT INNER APPEND LOG SIZE](this={palf_id:1008, self:"10.191.69.51:2882", has_set_deleted:false}, accum_size=1220) [2024-06-25 17:00:24.559570] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=5][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:24.559581] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=11][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:24.559585] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b4fb217b700, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024505658, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.559603] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=18][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b4fb217b700, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:3221638620, assoc_session_id:3221638620, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024505658, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.559616] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999990, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:24.559621] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:24.559626] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:24.559628] WDIAG [SQL] open (ob_result_set.cpp:157) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:24.559631] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=2][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:24.559635] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:24.559639] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=35, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:24.559646] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [63185][T1002_TeRec][T1001][YB420ABF4533-00061660F99EA31A-0-0] [lt=6] will sleep(sleep_us=35000, remain_us=27471790, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=35, v.err_=-4383, timeout_timestamp=1719306052031435) [2024-06-25 17:00:24.562509] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:24.562520] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:4000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.562529] WDIAG [SQL.DAS] get_das_task_id (ob_data_access_service.cpp:133) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=0][errcode=-4383] get das id not ready(ret=-4383, total_sleep_time=255000, max_sleep_time=200000) [2024-06-25 17:00:24.562534] WDIAG [SQL.DAS] create_das_task (ob_das_ref.cpp:512) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] get das task id failed(ret=-4383, ret="OB_GTI_NOT_READY") [2024-06-25 17:00:24.562539] WDIAG [SQL.ENG] create_one_das_task (ob_table_scan_op.cpp:665) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=5][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:24.562543] WDIAG [SQL.ENG] prepare_das_task (ob_table_scan_op.cpp:743) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] create one das task failed(ret=-4383) [2024-06-25 17:00:24.562547] WDIAG [SQL.ENG] prepare_all_das_tasks (ob_table_scan_op.cpp:816) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:24.562552] WDIAG [SQL.ENG] do_init_before_get_row (ob_table_scan_op.cpp:1384) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] prepare das task failed(ret=-4383) [2024-06-25 17:00:24.562555] WDIAG [SQL.ENG] inner_get_next_batch_for_tsc (ob_table_scan_op.cpp:2015) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=2][errcode=-4383] failed to init before get row(ret=-4383) [2024-06-25 17:00:24.562558] WDIAG [SQL.ENG] inner_get_next_batch (ob_table_scan_op.cpp:2000) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:24.562563] WDIAG [SQL.ENG] get_next_batch (ob_operator.cpp:1194) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] get next batch failed(ret=-4383, eval_ctx={batch_idx:0, batch_size:16, max_batch_size:16, frames_:0x2babb4414380}, id=2, op_name="PHY_TABLE_SCAN") [2024-06-25 17:00:24.562571] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1535) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=7][errcode=-4383] get next batch failed(ret=-4383) [2024-06-25 17:00:24.562574] WDIAG [SQL.ENG] get_next_row_vectorizely (ob_operator.cpp:1463) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=2][errcode=-4383] Failed to get_next_row vectorizedly(ret=-4383) [2024-06-25 17:00:24.562579] WDIAG [SQL.EXE] inner_get_next_row (ob_direct_transmit_op.cpp:27) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] get next row from child failed(ret=-4383) [2024-06-25 17:00:24.562582] WDIAG [SQL.ENG] get_next_row (ob_operator.cpp:1088) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=2][errcode=-4383] inner get next row failed(ret=-4383, type=14, op="PHY_DIRECT_TRANSMIT", op_id=1) [2024-06-25 17:00:24.562587] WDIAG [SQL.EXE] sync_send_result (ob_remote_executor_processor.cpp:283) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] failed to get next row(ret=-4383) [2024-06-25 17:00:24.562591] WDIAG [SQL.EXE] execute_remote_plan (ob_remote_executor_processor.cpp:469) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=4][errcode=-4383] sync send result failed(ret=-4383) [2024-06-25 17:00:24.562598] WDIAG [SQL.EXE] process (ob_remote_executor_processor.cpp:969) [12536][T1002_L0_G0][T1002][YB420ABA45B0-0006164506904EEC-0-0] [lt=3][errcode=-4383] execute remote plan failed(ret=-4383, local_tenant_schema_version=1718209168713032, exec_ctx_.get_task_exec_ctx().get_query_tenant_begin_schema_version()=1718209168713032) [2024-06-25 17:00:24.562847] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62234][pnio1][T0][YB420ABDC060-0006164325C301AB-0-0] [lt=9][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5377, fly_ts=22301563, send_timestamp=1719306002261283) [2024-06-25 17:00:24.563130] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62238][pnio1][T0][YB420ABF4538-00061669548FCCC7-0-0] [lt=6][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5398, fly_ts=22301556, send_timestamp=1719306002261573) [2024-06-25 17:00:24.563176] INFO [LIB] stat (utility.h:1197) [63266][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=3] [REPLAY STAT REPLAY TASK IN QUEUE TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=8, this=0x2b3a872048f8, extra_info="") [2024-06-25 17:00:24.563184] INFO [LIB] stat (utility.h:1197) [63266][T1002_ReplaySrv][T1002][Y0-0000000000000000-0-0] [lt=8] [REPLAY STAT REPLAY TASK EXECUTE COST TIME](cur_stat_count=39, stat_interval=1000000, avg (count/cost)=1, this=0x2b3a87204b28, extra_info="") [2024-06-25 17:00:24.566276] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F4905-0-0] [lt=5][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=2208, pos=316, bitmap=NULL) [2024-06-25 17:00:24.566457] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F4905-0-0] [lt=10][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=75, len=428, pos=303, bitmap=NULL) [2024-06-25 17:00:24.566660] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F4905-0-0] [lt=6][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=16, len=219, pos=190, bitmap=NULL) [2024-06-25 17:00:24.566861] WDIAG [RPC.OBMYSQL] serialize (ob_mysql_row.cpp:56) [62927][T1_L0_G0][T1][YB420ABF4533-000616614A8F4905-0-0] [lt=4][errcode=-4019] failed to encode cell(ret=-4019, cell_idx=1, len=38, pos=17, bitmap=NULL) [2024-06-25 17:00:24.567959] WDIAG [STORAGE.TRANS] gen_trans_id (ob_trans_service_v4.cpp:2448) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4383] get trans id not ready(ret=-4383, retry_times=52, this={is_inited_:true, tenant_id_:1001, this:0x2b3a99c04030}) [2024-06-25 17:00:24.567969] WDIAG [STORAGE.TRANS] add (ob_trans_define_v4.cpp:1492) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=10][errcode=-4383] fail to exec tx_id_allocator_(tx_id)(ret=-4383) [2024-06-25 17:00:24.567973] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:261) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=4][errcode=-4383] add tx to txMgr fail(ret=-4383, tx={this:0x2b3a78df0520, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024514059, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.567990] WDIAG [STORAGE.TRANS] start_tx (ob_tx_api.cpp:287) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=17][errcode=-4383] start tx failed(ret=-4383, tx={this:0x2b3a78df0520, tx_id:{txid:0}, state:1, addr:"10.191.69.51:2882", tenant_id:1001, session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"", xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0, isolation:1, snapshot_version:{val:18446744073709551615, v:3}, snapshot_scn:0, active_scn:0, op_sn:2, alloc_ts:1719306024514059, active_ts:-1, commit_ts:-1, finish_ts:-1, timeout_us:-1, lock_timeout_us:-1, expire_ts:9223372036854775807, coord_id:{id:-1}, parts:[], exec_info_reap_ts:0, commit_version:{val:18446744073709551615, v:3}, commit_times:0, commit_cb:null, cluster_id:-1, cluster_version:0, flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false, flags_.REPLICA:false, can_elr:false, cflict_txs:[], abort_cause:0, commit_expire_ts:0, commit_task_.is_registered():false, ref:1}) [2024-06-25 17:00:24.568003] WDIAG [SQL.EXE] explicit_start_trans (ob_sql_trans_control.cpp:175) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=12][errcode=-4383] fail to exec txs->start_tx(*session->get_tx_desc(), tx_param)(ret=-4383, tx_param={cluster_id:1713323324, timeout_us:29999992, lock_timeout_us:-1, access_mode:0, isolation:1}) [2024-06-25 17:00:24.568009] WDIAG [SQL.ENG] start_trans (ob_tcl_executor.cpp:110) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=4][errcode=-4383] fail start trans(ret=-4383) [2024-06-25 17:00:24.568013] WDIAG [SQL] open_cmd (ob_result_set.cpp:98) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4383] execute cmd failed(ret=-4383) [2024-06-25 17:00:24.568015] WDIAG [SQL] open (ob_result_set.cpp:157) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=2][errcode=-4383] execute plan failed(ret=-4383) [2024-06-25 17:00:24.568019] WDIAG [SERVER] open (ob_inner_sql_result.cpp:153) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=4][errcode=-4383] open result set failed(ret=-4383) [2024-06-25 17:00:24.568022] WDIAG [SERVER] do_query (ob_inner_sql_connection.cpp:768) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=2][errcode=-4383] result set open failed(ret=-4383, executor={ObIExecutor:, sql:"START TRANSACTION"}) [2024-06-25 17:00:24.568026] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:913) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=4][errcode=-4383] execute failed(ret=-4383, tenant_id=1001, executor={ObIExecutor:, sql:"START TRANSACTION"}, retry_cnt=14, local_sys_schema_version=1719272886792320, local_tenant_schema_version=1713325430335672) [2024-06-25 17:00:24.568032] INFO [SERVER] sleep_before_local_retry (ob_query_retry_ctrl.cpp:91) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=6] will sleep(sleep_us=14000, remain_us=29100855, base_sleep_us=1000, retry_sleep_type=1, v.stmt_retry_times_=14, v.err_=-4383, timeout_timestamp=1719306053668887) [2024-06-25 17:00:24.569448] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1006}) [2024-06-25 17:00:24.569455] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=6] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1007}) [2024-06-25 17:00:24.569457] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1008}) [2024-06-25 17:00:24.569460] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=2] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1009}) [2024-06-25 17:00:24.569462] INFO [STORAGE.TRANS] check_all_readonly_tx_clean_up (ob_ls_tx_service.cpp:264) [63310][T1002_TxLoopWor][T1002][Y0-0000000000000000-0-0] [lt=1] wait_all_readonly_tx_cleaned_up cleaned up success(ls_id={id:1010}) [2024-06-25 17:00:24.569667] WDIAG [RPC] create (ob_poc_rpc_server.cpp:112) [62241][pnio1][T0][YB420ABDC060-000616432547F0CA-0-0] [lt=8][errcode=0] PNIO packet wait too much time between proxy and server_cb(pcode=5378, fly_ts=22301558, send_timestamp=1719306002268108) [2024-06-25 17:00:24.580051] WDIAG [STORAGE.TRANS] process_cluster_heartbeat_rpc_cb (ob_tenant_weak_read_service.cpp:452) [62243][pnio1][T1001][YB420ABF4533-00061661718F6568-0-0] [lt=9][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:24.582094] WDIAG [SERVER] after_func (ob_query_retry_ctrl.cpp:935) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=6][errcode=-4383] [RETRY] check if need retry(v={force_local_retry:true, stmt_retry_times:14, local_retry_times:14, err_:-4383, err_:"OB_GTI_NOT_READY", retry_type:1, client_ret:-4383}, need_retry=true) [2024-06-25 17:00:24.582103] WDIAG [SERVER] inner_close (ob_inner_sql_result.cpp:222) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=8][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.582106] WDIAG [SERVER] force_close (ob_inner_sql_result.cpp:202) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=3][errcode=-4383] result set close failed(ret=-4383) [2024-06-25 17:00:24.582108] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:918) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=2][errcode=-4383] failed to close result(close_ret=-4383, ret=-4383) [2024-06-25 17:00:24.582113] WDIAG [SERVER] query (ob_inner_sql_connection.cpp:948) [62272][TbltTblUp0][T1001][YB420ABF4533-000616423D1F16A8-0-0] [lt=2][errcode=-4383] failed to process record(executor={ObIExecutor:, sql:"START TRANSACTION"}, record_ret=-4383, ret=-4383) [2024-06-25 17:00:24.583067] WDIAG [RPC] send (ob_poc_rpc_proxy.h:161) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0][errcode=-4013] REACH SYSLOG RATE LIMIT [bandwidth] [2024-06-25 17:00:24.583079] INFO [SQL.DAS] fetch_new_range (ob_das_id_rpc.cpp:114) [12807][T1002_L0_G0][T1002][YB420ABDC060-0006164245661A9E-0-0] [lt=0] fetch new DAS ID range finish(ret=-4013, ret="OB_ALLOCATE_MEMORY_FAILED", msg={tenant_id:1002, range:3000000}, res={tenant_id:0, status:0, start_id:0, end_id:0}) [2024-06-25 17:00:24.584983] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1457) [95800][T1004_L0_G1][T1004][YB420ABA45B1-00061646124EF5DF-0-0] [lt=2] receive register_req from acceptor, ignore(self="10.191.69.51:2882", palf_id=1039, child={server:"10.186.69.177:2882", region:"sys_region", register_time_us:1719306024585395, keepalive_ts:-1}) [2024-06-25 17:00:24.584992] INFO [PALF] handle_register_parent_req (palf_handle_impl.cpp:1476) [95800][T1004_L0_G1][T1004][YB420ABA45B1-00061646124EF5DF-0-0] [lt=8] handle_register_parent_req finished(ret=0, ret="OB_SUCCESS", palf_id=1039, self="10.191.69.51:2882", child={server:"10.186.69.177:2882", region:"sys_region", register_time_us:1719306024585395, keepalive_ts:-1}, is_to_leader=true)