2023-12-06 11:30:05.818 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.c.t.e.runner.JavaSubtaskRunner : Run subtask, id=4013106, context=Context{parallelIdx=-1, stringMap={checkBeforeDataBackup=true, ob_version=4.2.1.1, startClusterCompact=false, task_instance_id=4011114, dataBackupTriggerInfo={"async":true,"backupDimension":"CLUSTER","dataBackupMode":"FULL_BACKUP","triggerTime":"2023-12-06T03:30:04Z","triggerType":"BACKUP_NOW"}, task_operation=execute, triggerTime=1701833404000000, ob_cluster_id=1, startLogBackup=true, backupDim=CLUSTER, dataSetIncIdMap={"1002":"1000003"}, cluster_id=1, storageBaseInfo=******, tenantName=*, backupMode=physical, triggerType=BACKUP_NOW, latest_execution_start_time=2023-12-06T11:30:05.781+08:00, sub_task_instance_name=Wait data backup task successful, backupNow=true, sub_task_instance_id=4013106, dataBackupMode=FULL_BACKUP, obClusterName=obcluster1}, listMap={}}, executor=10.10.10.148 2023-12-06 11:30:05.821 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=1 2023-12-06 11:30:05.837 WARN 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.s.o.o.f.ConnectPropertiesBuilder : Local odp client is not connectable, so use direct mode to connect OB 2023-12-06 11:30:05.840 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.10.10.145, sql port:2881 2023-12-06 11:30:05.842 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.10.10.145, sql port:2881 2023-12-06 11:30:05.843 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2023-12-06 11:30:05.845 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT * FROM (SELECT job_id, incarnation, job.tenant_id as tenant_id, job.backup_set_id as backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'TENANT' as backup_level, file.MIN_RESTORE_SCN_DISPLAY as snapshot_version_time FROM ( SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, NULL AS end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) AS job LEFT JOIN (SELECT tenant_id AS tid, backup_set_id AS sid, min_restore_scn_display FROM CDB_OB_BACKUP_SET_FILES) AS file ON job.tenant_id = file.tid AND job.backup_set_id = file.sid) WHERE tenant_id != 1 AND description = ? ORDER BY start_time DESC) where start_time >= '2023-12-06 11:30:04', args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2023-12-06 11:30:04.0] 2023-12-06 11:30:05.877 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task status: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=164, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1002, clusterType=null, tenantId=1002, tenantName=null, snapshotVersionTime=null, startTime=2023-12-06T03:30:04.862689Z, finishTime=null, progress=null, status=INITIALIZING, errorMsg=, dataVersion=null, backupUri=file:///data/nfs/obbackup/obcluster1/1/tenant_incarnation_1/1002/data, duration=null, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=164, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=TENANT, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2023-12-06 11:30:15.896 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=2 2023-12-06 11:30:16.327 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obcluster1, tenantName=sys, dbUser=ocp_monitor 2023-12-06 11:30:16.330 WARN 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.s.o.o.f.ConnectPropertiesBuilder : Local odp client is not connectable, so use direct mode to connect OB 2023-12-06 11:30:16.332 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.10.10.145, sql port:2881 2023-12-06 11:30:16.336 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.10.10.145, sql port:2881 2023-12-06 11:30:16.337 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2023-12-06 11:30:16.340 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT * FROM (SELECT job_id, incarnation, job.tenant_id as tenant_id, job.backup_set_id as backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'TENANT' as backup_level, file.MIN_RESTORE_SCN_DISPLAY as snapshot_version_time FROM ( SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, NULL AS end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) AS job LEFT JOIN (SELECT tenant_id AS tid, backup_set_id AS sid, min_restore_scn_display FROM CDB_OB_BACKUP_SET_FILES) AS file ON job.tenant_id = file.tid AND job.backup_set_id = file.sid) WHERE tenant_id != 1 AND description = ? ORDER BY start_time DESC) where start_time >= '2023-12-06 11:30:04', args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2023-12-06 11:30:04.0] 2023-12-06 11:30:16.353 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task status: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=164, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1002, clusterType=null, tenantId=1002, tenantName=null, snapshotVersionTime=null, startTime=2023-12-06T03:30:04.862689Z, finishTime=null, progress=null, status=INITIALIZING, errorMsg=, dataVersion=null, backupUri=file:///data/nfs/obbackup/obcluster1/1/tenant_incarnation_1/1002/data, duration=null, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=164, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=TENANT, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2023-12-06 11:30:26.357 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=3 2023-12-06 11:30:26.398 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obcluster1, tenantName=sys, dbUser=ocp_monitor 2023-12-06 11:30:26.403 WARN 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.s.o.o.f.ConnectPropertiesBuilder : Local odp client is not connectable, so use direct mode to connect OB 2023-12-06 11:30:26.408 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.10.10.145, sql port:2881 2023-12-06 11:30:26.411 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.10.10.145, sql port:2881 2023-12-06 11:30:26.413 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2023-12-06 11:30:26.418 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT * FROM (SELECT job_id, incarnation, job.tenant_id as tenant_id, job.backup_set_id as backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'TENANT' as backup_level, file.MIN_RESTORE_SCN_DISPLAY as snapshot_version_time FROM ( SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, NULL AS end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) AS job LEFT JOIN (SELECT tenant_id AS tid, backup_set_id AS sid, min_restore_scn_display FROM CDB_OB_BACKUP_SET_FILES) AS file ON job.tenant_id = file.tid AND job.backup_set_id = file.sid) WHERE tenant_id != 1 AND description = ? ORDER BY start_time DESC) where start_time >= '2023-12-06 11:30:04', args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2023-12-06 11:30:04.0] 2023-12-06 11:30:26.437 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task status: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=164, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1002, clusterType=null, tenantId=1002, tenantName=null, snapshotVersionTime=null, startTime=2023-12-06T03:30:04.862689Z, finishTime=2023-12-06T03:30:23.304231Z, progress=null, status=FAILED, errorMsg=(ROOTSERVICE)addr: 10.10.10.146:2882, result: -9080(the format file does not exist under the destination), trace_id: YB420A0A0A92-00060BCEE06A8775-0-0, dataVersion=null, backupUri=file:///data/nfs/obbackup/obcluster1/1/tenant_incarnation_1/1002/data, duration=19, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=164, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=TENANT, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2023-12-06 11:30:26.457 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, userId=200, clusterName=obcluster1, tenantName=sys, dbUser=ocp_monitor 2023-12-06 11:30:26.459 WARN 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.s.o.o.f.ConnectPropertiesBuilder : Local odp client is not connectable, so use direct mode to connect OB 2023-12-06 11:30:26.461 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.10.10.145, sql port:2881 2023-12-06 11:30:26.464 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ObConnectors : [obsdk]:connected server ip:10.10.10.145, sql port:2881 2023-12-06 11:30:26.466 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2023-12-06 11:30:26.469 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT * FROM (SELECT * FROM (SELECT job_id, incarnation, job.tenant_id as tenant_id, job.backup_set_id as backup_set_id, backup_type, path AS backup_dest, start_timestamp as start_time, end_timestamp as end_time, now(6) as check_time, status, comment, description, 'TENANT' as backup_level, file.MIN_RESTORE_SCN_DISPLAY as snapshot_version_time FROM ( SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, NULL AS end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) AS job LEFT JOIN (SELECT tenant_id AS tid, backup_set_id AS sid, min_restore_scn_display FROM CDB_OB_BACKUP_SET_FILES) AS file ON job.tenant_id = file.tid AND job.backup_set_id = file.sid) WHERE tenant_id != 1 AND description = ? ORDER BY start_time DESC) where start_time >= '2023-12-06 11:30:04', args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2023-12-06 11:30:04.0] 2023-12-06 11:30:26.489 INFO 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task result: [DataBackupHistoryWrapperTask(super=DataBackupHistoryTask(taskId=164, backupType=FULL, taskType=null, taskCount=null, clusterName=null, clusterId=null, obClusterId=null, obTenantId=1002, clusterType=null, tenantId=1002, tenantName=null, snapshotVersionTime=null, startTime=2023-12-06T03:30:04.862689Z, finishTime=2023-12-06T03:30:23.304231Z, progress=null, status=FAILED, errorMsg=(ROOTSERVICE)addr: 10.10.10.146:2882, result: -9080(the format file does not exist under the destination), trace_id: YB420A0A0A92-00060BCEE06A8775-0-0, dataVersion=null, backupUri=file:///data/nfs/obbackup/obcluster1/1/tenant_incarnation_1/1002/data, duration=19, backupSize=null, dataType=null, schemaVersion=null, lastSuccessTime=null, backupSetId=164, backupLevel=null), backupMode=PHYSICAL_BACKUP, dataBackupTaskType=FULL_BACKUP, backupDimension=TENANT, backupSizeUnit=null, backupSetFile=null, backupSetFilePath=null, dataSecondaryBackupTaskIds=null)] 2023-12-06 11:30:26.508 ERROR 3013778 --- [pool-manual-subtask-executor16,2c7242d142b94de6,de80fdecd61f] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : null java.lang.NullPointerException: null at com.oceanbase.ocp.backup.internal.data.meta.PhysicalBackupFileMetaManager.updateClusterDataSetCheckpoint(PhysicalBackupFileMetaManager.java:111) at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.lambda$checkDataBackupTaskResult$3(WaitDataBackupSuccessTask.java:140) at java.util.ArrayList.forEach(ArrayList.java:1259) at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.checkDataBackupTaskResult(WaitDataBackupSuccessTask.java:134) at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.run(WaitDataBackupSuccessTask.java:205) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.execute(JavaSubtaskRunner.java:59) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:31) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:25) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:193) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:187) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:124) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 4013106, operation:EXECUTE, state: FAILED