############{EXECUTE}{2022-12-09T14:15:48.939+08:00}############ 2022-12-09 14:15:49.294 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.t.e.runner.JavaSubtaskRunner : Run subtask, id=14973, context=Context{parallelIdx=-1, stringMap={checkBeforeDataBackup=true, ob_version=4.0.0.0, prohibit_rollback=false, startClusterCompact=false, task_instance_id=14759, task_operation=execute, triggerTime=1670566543099000, ob_cluster_id=1, startLogBackup=true, backupDim=CLUSTER, dataSetIncIdMap={"1006":"2","1004":"1"}, cluster_id=1, storageBaseInfo=******, tenantName=*, backupMode=physical, triggerType=BACKUP_NOW, latest_execution_start_time=2022-12-09T14:15:48.722+08:00, backupNow=true, sub_task_instance_id=14973, dataBackupMode=FULL_BACKUP, obClusterName=obcluster}, listMap={}}, executor=192.168.28.62 2022-12-09 14:15:49.407 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=1 2022-12-09 14:15:49.723 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:15:49.917 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:15:50.106 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2022-12-09 14:15:50.191 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT incarnation, tenant_id, backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'CLUSTER' as backup_level FROM ( SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) WHERE tenant_id <> 1 AND description = ?) where start_time >= '2022-12-09 14:15:43' ORDER BY start_time DESC, args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2022-12-09 14:15:43.099] 2022-12-09 14:15:50.553 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task status: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1006, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.611367Z, finishTime=null, status=DOING, errorMsg=, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1006/data, duration=null, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null), DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1004, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.338416Z, finishTime=null, status=DOING, errorMsg=, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1004/data, duration=null, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2022-12-09 14:16:01.216 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=2 2022-12-09 14:16:01.745 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obcluster, tenantName=sys, dbUser=ocp_monitor 2022-12-09 14:16:01.901 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:16:02.068 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:16:02.257 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2022-12-09 14:16:02.363 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT incarnation, tenant_id, backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'CLUSTER' as backup_level FROM ( SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) WHERE tenant_id <> 1 AND description = ?) where start_time >= '2022-12-09 14:15:43' ORDER BY start_time DESC, args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2022-12-09 14:15:43.099] 2022-12-09 14:16:02.598 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task status: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1006, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.611367Z, finishTime=null, status=DOING, errorMsg=, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1006/data, duration=null, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null), DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1004, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.338416Z, finishTime=null, status=DOING, errorMsg=, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1004/data, duration=null, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2022-12-09 14:16:12.798 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=3 2022-12-09 14:16:13.374 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obcluster, tenantName=sys, dbUser=ocp_monitor 2022-12-09 14:16:13.836 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:16:14.076 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:16:14.581 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2022-12-09 14:16:14.688 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT incarnation, tenant_id, backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'CLUSTER' as backup_level FROM ( SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) WHERE tenant_id <> 1 AND description = ?) where start_time >= '2022-12-09 14:15:43' ORDER BY start_time DESC, args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2022-12-09 14:15:43.099] 2022-12-09 14:16:15.122 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task status: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1006, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.611367Z, finishTime=2022-12-09T06:16:13.673491Z, status=FAILED, errorMsg=cannot find backup file, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1006/data, duration=28, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null), DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1004, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.338416Z, finishTime=null, status=DOING, errorMsg=, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1004/data, duration=null, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2022-12-09 14:16:25.685 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=4 2022-12-09 14:16:25.977 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obcluster, tenantName=sys, dbUser=ocp_monitor 2022-12-09 14:16:26.157 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:16:26.310 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:16:26.403 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2022-12-09 14:16:26.506 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT incarnation, tenant_id, backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'CLUSTER' as backup_level FROM ( SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) WHERE tenant_id <> 1 AND description = ?) where start_time >= '2022-12-09 14:15:43' ORDER BY start_time DESC, args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2022-12-09 14:15:43.099] 2022-12-09 14:16:26.674 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task status: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1006, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.611367Z, finishTime=2022-12-09T06:16:13.673491Z, status=FAILED, errorMsg=cannot find backup file, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1006/data, duration=28, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null), DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1004, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.338416Z, finishTime=null, status=DOING, errorMsg=, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1004/data, duration=null, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2022-12-09 14:16:36.738 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=5 2022-12-09 14:16:36.950 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obcluster, tenantName=sys, dbUser=ocp_monitor 2022-12-09 14:16:36.994 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:16:37.054 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:16:37.099 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2022-12-09 14:16:37.215 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT incarnation, tenant_id, backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'CLUSTER' as backup_level FROM ( SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) WHERE tenant_id <> 1 AND description = ?) where start_time >= '2022-12-09 14:15:43' ORDER BY start_time DESC, args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2022-12-09 14:15:43.099] 2022-12-09 14:16:37.314 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task status: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1006, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.611367Z, finishTime=2022-12-09T06:16:13.673491Z, status=FAILED, errorMsg=cannot find backup file, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1006/data, duration=28, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null), DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1004, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.338416Z, finishTime=2022-12-09T06:16:29.947443Z, status=FAILED, errorMsg=cannot find backup file, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1004/data, duration=44, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2022-12-09 14:16:37.461 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obcluster, tenantName=sys, dbUser=ocp_monitor 2022-12-09 14:16:37.503 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:16:37.547 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:16:37.628 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2022-12-09 14:16:37.711 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT incarnation, tenant_id, backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'CLUSTER' as backup_level FROM ( SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) WHERE tenant_id <> 1 AND description = ?) where start_time >= '2022-12-09 14:15:43' ORDER BY start_time DESC, args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2022-12-09 14:15:43.099] 2022-12-09 14:16:37.819 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task result: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1006, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.611367Z, finishTime=2022-12-09T06:16:13.673491Z, status=FAILED, errorMsg=cannot find backup file, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1006/data, duration=28, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null), DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1004, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.338416Z, finishTime=2022-12-09T06:16:29.947443Z, status=FAILED, errorMsg=cannot find backup file, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1004/data, duration=44, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2022-12-09 14:16:38.746 WARN 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,1b5941b1ac0f] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceOverview{id=14973, name=Wait data backup task successful, state=FAILED, operation=EXECUTE, className=com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask, seriesId=1, startTime=2022-12-09T14:15:48.723+08:00, endTime=null}, failedMessage=Data backup task failed. com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=BACKUP_DATA_BACKUP_TASK_FAILED, args= at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.lambda$checkDataBackupTaskResult$3(WaitDataBackupSuccessTask.java:142) ~[ocp-backup-service-4.0.0-20220930.jar!/:4.0.0-20220930] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[na:1.8.0_312] at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.checkDataBackupTaskResult(WaitDataBackupSuccessTask.java:132) ~[ocp-backup-service-4.0.0-20220930.jar!/:4.0.0-20220930] at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.run(WaitDataBackupSuccessTask.java:201) ~[ocp-backup-service-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.JavaSubtaskRunner.execute(JavaSubtaskRunner.java:60) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:32) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:111) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(RunnerFactory.java:183) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.Run nerFactory.run(RunnerFactory.java:101) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$null$1(ReadySubtaskWorker.java:117) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Set state for subtask: 14973, operation:EXECUTE, state: FAILED ############{RETRY}{2022-12-09T14:18:28.671+08:00}############ 2022-12-09 14:18:28.894 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.a.o.c.t.e.runner.JavaSubtaskRunner : Retry subtask, id=14973, context=Context{parallelIdx=-1, stringMap={checkBeforeDataBackup=true, ob_version=4.0.0.0, prohibit_rollback=false, startClusterCompact=false, task_instance_id=14759, task_operation=retry, triggerTime=1670566543099000, ob_cluster_id=1, startLogBackup=true, backupDim=CLUSTER, dataSetIncIdMap={"1006":"2","1004":"1"}, cluster_id=1, storageBaseInfo=******, tenantName=*, backupMode=physical, triggerType=BACKUP_NOW, latest_execution_start_time=2022-12-09T14:18:28.583+08:00, backupNow=true, sub_task_instance_id=14973, dataBackupMode=FULL_BACKUP, obClusterName=obcluster}, listMap={}}, executor=192.168.28.62 2022-12-09 14:18:28.965 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : do nothing in rollback! 2022-12-09 14:18:29.056 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=1 2022-12-09 14:18:29.374 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:18:29.557 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:18:29.667 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2022-12-09 14:18:29.763 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT incarnation, tenant_id, backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'CLUSTER' as backup_level FROM ( SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) WHERE tenant_id <> 1 AND description = ?) where start_time >= '2022-12-09 14:15:43' ORDER BY start_time DESC, args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2022-12-09 14:15:43.099] 2022-12-09 14:18:30.003 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task status: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1006, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.611367Z, finishTime=2022-12-09T06:16:13.673491Z, status=FAILED, errorMsg=cannot find backup file, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1006/data, duration=28, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null), DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1004, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.338416Z, finishTime=2022-12-09T06:16:29.947443Z, status=FAILED, errorMsg=cannot find backup file, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1004/data, duration=44, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2022-12-09 14:18:30.513 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.a.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obcluster, tenantName=sys, dbUser=ocp_monitor 2022-12-09 14:18:30.601 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:18:30.707 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.a.o.core.obsdk.connector.ObConnectors : [obsdk]:connected server ip:192.168.28.57, sql port:2881 2022-12-09 14:18:30.777 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2022-12-09 14:18:30.852 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.a.o.c.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT incarnation, tenant_id, backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'CLUSTER' as backup_level FROM ( SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) WHERE tenant_id <> 1 AND description = ?) where start_time >= '2022-12-09 14:15:43' ORDER BY start_time DESC, args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2022-12-09 14:15:43.099] 2022-12-09 14:18:31.029 INFO 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task result: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1006, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.611367Z, finishTime=2022-12-09T06:16:13.673491Z, status=FAILED, errorMsg=cannot find backup file, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1006/data, duration=28, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null), DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=1, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1004, clusterType=null, tenantId=null, tenantName=null, startTime=2022-12-09T06:15:45.338416Z, finishTime=2022-12-09T06:16:29.947443Z, status=FAILED, errorMsg=cannot find backup file, dataVersion=null, backupUri=file:///data/obcluster/1/tenant_incarnation_1/1004/data, duration=44, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=1, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=CLUSTER, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2022-12-09 14:18:31.483 WARN 42 --- [pool-manual-subtask-executor13,831eed6db5ce44c6,6bfb2a01e682] c.a.o.c.t.engine.runner.RunnerFactory : Execute task failed, subtask=SubtaskInstanceOverview{id=14973, name=Wait data backup task successful, state=FAILED, operation=RETRY, className=com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask, seriesId=1, startTime=2022-12-09T14:15:48.723+08:00, endTime=null}, failedMessage=Data backup task failed. com.alipay.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=BACKUP_DATA_BACKUP_TASK_FAILED, args= at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.lambda$checkDataBackupTaskResult$3(WaitDataBackupSuccessTask.java:142) ~[ocp-backup-service-4.0.0-20220930.jar!/:4.0.0-20220930] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[na:1.8.0_312] at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.checkDataBackupTaskResult(WaitDataBackupSuccessTask.java:132) ~[ocp-backup-service-4.0.0-20220930.jar!/:4.0.0-20220930] at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.run(WaitDataBackupSuccessTask.java:201) ~[ocp-backup-service-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.runtime.Subtask.retry(Subtask.java:49) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.JavaSubtaskRunner.retry(JavaSubtaskRunner.java:74) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:35) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:111) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.redirectOutputIfNotSysSchedule(Runner Factory.java:183) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.runner.RunnerFactory.run(RunnerFactory.java:101) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at com.alipay.ocp.core.task.engine.coordinator.worker.subtask.ReadySubtaskWorker.lambda$null$1(ReadySubtaskWorker.java:117) ~[ocp-core-4.0.0-20220930.jar!/:4.0.0-20220930] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_312] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_312] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_312] Set state for subtask: 14973, operation:RETRY, state: FAILED