2025-03-11 16:20:16.436 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.service.iam.user.UserService : user 100 login with organization 10000000 2025-03-11 16:20:16.441 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.c.t.e.runner.JavaSubtaskRunner : Run subtask, id=5556, context=Context{parallelIdx=-1, stringMap={checkBeforeDataBackup=true, ob_version=4.3.5.0, startClusterCompact=false, task_instance_id=5221, logBackupRecordList=[{"clusterId":2,"obTenantId":1002,"status":"STOPPED","tenantId":5,"tenantName":"testtenant"}], dataBackupTriggerInfo={"async":true,"backupDimension":"CLUSTER","dataBackupMode":"FULL_BACKUP","triggerTime":"2025-03-11T08:20:15Z","triggerType":"BACKUP_NOW"}, description=ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2025-03-11 16:20:15.0, task_operation=execute, triggerTime=1741681215000000, ob_cluster_id=1741079861, startLogBackup=true, backupDim=CLUSTER, dataSetIncIdMap={"1002":"6"}, cluster_id=2, storageBaseInfo=******, tenantName=*, backupMode=physical, triggerType=BACKUP_NOW, latest_execution_start_time=2025-03-11T16:20:16.422+08:00, sub_task_instance_name=Wait data backup task successful, backupNow=true, sub_task_instance_id=5556, dataBackupMode=FULL_BACKUP, obClusterName=test}, listMap={}}, executor=10.1.46.60 2025-03-11 16:20:16.445 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=1 2025-03-11 16:20:16.462 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:20:16.467 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:20:16.472 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:20:16.476 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:20:16.480 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-11 16:20:16.488 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SHOW VARIABLES LIKE ? 2025-03-11 16:20:16.494 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:20:16.498 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:20:16.502 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-11 16:20:16.506 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT tenant_id, name, value, gmt_create, gmt_modified FROM __all_virtual_sys_variable WHERE name = ? 2025-03-11 16:20:16.522 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:20:16.526 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:20:16.530 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-11 16:20:16.560 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select * from (SELECT job_id, incarnation, job.tenant_id AS tenant_id, job.backup_set_id AS backup_set_id, backup_type, path AS backup_dest, start_timestamp AS start_time, end_timestamp AS end_time, now(6) AS check_time, task.data_progress AS data_progress, status, comment, description, 'TENANT' AS backup_level, file.min_restore_scn_display AS snapshot_version_time, output_bytes FROM ( SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, NULL AS end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) job LEFT JOIN (SELECT tenant_id AS tid, backup_set_id, min_restore_scn_display, output_bytes FROM CDB_OB_BACKUP_SET_FILES) file ON job.tenant_id = file.tid AND job.backup_set_id = file.backup_set_id LEFT JOIN (SELECT tenant_id AS tid, backup_set_id, data_progress FROM CDB_OB_BACKUP_TASKS) task ON job.tenant_id = task.tid AND job.backup_set_id = task.backup_set_id WHERE tenant_id != 1 AND description = ? ORDER BY start_time DESC) where start_time >= '2025-03-11 16:20:15', args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2025-03-11 16:20:15.0] 2025-03-11 16:20:16.625 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task: [DataBackupHistoryWrapperTask{taskId=10, backupType=FULL, clusterId=null, obTenantId=1002, backupSetId=10, backupDest=file:///data/nfs/test/1741079861/tenant_incarnation_1/1002/data, snapshotVersionTime=null, startTime=2025-03-11T08:20:15.562232Z, finishTime=null, status=INITIALIZING, progress=null, errorMsg=}] 2025-03-11 16:20:16.633 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : task status is UNKNOWN 2025-03-11 16:20:16.637 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : count running backup task size: 1 2025-03-11 16:20:46.642 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=2 2025-03-11 16:20:46.661 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=test, tenantName=sys, dbUser=ocp_monitor 2025-03-11 16:20:46.665 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:20:46.670 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:20:46.679 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:20:46.683 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:20:46.687 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-11 16:20:46.692 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SHOW VARIABLES LIKE ? 2025-03-11 16:20:46.698 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:20:46.702 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:20:46.707 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-11 16:20:46.712 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT tenant_id, name, value, gmt_create, gmt_modified FROM __all_virtual_sys_variable WHERE name = ? 2025-03-11 16:20:46.728 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:20:46.733 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:20:46.737 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-11 16:20:46.767 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select * from (SELECT job_id, incarnation, job.tenant_id AS tenant_id, job.backup_set_id AS backup_set_id, backup_type, path AS backup_dest, start_timestamp AS start_time, end_timestamp AS end_time, now(6) AS check_time, task.data_progress AS data_progress, status, comment, description, 'TENANT' AS backup_level, file.min_restore_scn_display AS snapshot_version_time, output_bytes FROM ( SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, NULL AS end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) job LEFT JOIN (SELECT tenant_id AS tid, backup_set_id, min_restore_scn_display, output_bytes FROM CDB_OB_BACKUP_SET_FILES) file ON job.tenant_id = file.tid AND job.backup_set_id = file.backup_set_id LEFT JOIN (SELECT tenant_id AS tid, backup_set_id, data_progress FROM CDB_OB_BACKUP_TASKS) task ON job.tenant_id = task.tid AND job.backup_set_id = task.backup_set_id WHERE tenant_id != 1 AND description = ? ORDER BY start_time DESC) where start_time >= '2025-03-11 16:20:15', args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2025-03-11 16:20:15.0] 2025-03-11 16:20:46.803 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task: [DataBackupHistoryWrapperTask{taskId=10, backupType=FULL, clusterId=null, obTenantId=1002, backupSetId=10, backupDest=file:///data/nfs/test/1741079861/tenant_incarnation_1/1002/data, snapshotVersionTime=null, startTime=2025-03-11T08:20:15.562232Z, finishTime=null, status=DOING, progress=null, errorMsg=}] 2025-03-11 16:20:46.815 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : count running backup task size: 1 2025-03-11 16:21:16.820 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : try to check data backup task finished, retryTimes=3 2025-03-11 16:21:16.834 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.s.o.o.f.ConnectPropertiesBuilder : get credential from obsdk context, clusterName=test, tenantName=sys, dbUser=ocp_monitor 2025-03-11 16:21:16.839 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:21:16.843 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:21:16.848 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:21:16.851 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:21:16.855 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-11 16:21:16.859 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SHOW VARIABLES LIKE ? 2025-03-11 16:21:16.864 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:21:16.868 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:21:16.872 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-11 16:21:16.876 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: SELECT tenant_id, name, value, gmt_create, gmt_modified FROM __all_virtual_sys_variable WHERE name = ? 2025-03-11 16:21:16.892 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.161, port=2881, database=oceanbase) 2025-03-11 16:21:16.896 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.obsdk.connector.ObConnectorHolder : [obsdk] no ob connector found in holder, key=ObConnectorKey(connectionMode=direct, clusterName=test, obClusterId=1741079861, tenantName=sys, username=ocp_monitor, address=10.1.46.124, port=2881, database=oceanbase) 2025-03-11 16:21:16.900 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: set ob_query_timeout = ?, args: [10000000] 2025-03-11 16:21:16.929 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.ocp.obsdk.connector.ConnectTemplate : [obsdk] sql: select * from (SELECT job_id, incarnation, job.tenant_id AS tenant_id, job.backup_set_id AS backup_set_id, backup_type, path AS backup_dest, start_timestamp AS start_time, end_timestamp AS end_time, now(6) AS check_time, task.data_progress AS data_progress, status, comment, description, 'TENANT' AS backup_level, file.min_restore_scn_display AS snapshot_version_time, output_bytes FROM ( SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, NULL AS end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOBS UNION SELECT job_id, incarnation, tenant_id, backup_set_id, backup_type, path, start_timestamp, end_timestamp, status, comment, description FROM CDB_OB_BACKUP_JOB_HISTORY ) job LEFT JOIN (SELECT tenant_id AS tid, backup_set_id, min_restore_scn_display, output_bytes FROM CDB_OB_BACKUP_SET_FILES) file ON job.tenant_id = file.tid AND job.backup_set_id = file.backup_set_id LEFT JOIN (SELECT tenant_id AS tid, backup_set_id, data_progress FROM CDB_OB_BACKUP_TASKS) task ON job.tenant_id = task.tid AND job.backup_set_id = task.backup_set_id WHERE tenant_id != 1 AND description = ? ORDER BY start_time DESC) where start_time >= '2025-03-11 16:20:15', args: [ALTER BACKUP CLUSTER FULL_BACKUP BY BACKUP_NOW AT 2025-03-11 16:20:15.0] 2025-03-11 16:21:16.962 INFO 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task: [DataBackupHistoryWrapperTask{taskId=10, backupType=FULL, clusterId=null, obTenantId=1002, backupSetId=10, backupDest=file:///data/nfs/test/1741079861/tenant_incarnation_1/1002/data, snapshotVersionTime=null, startTime=2025-03-11T08:20:15.562232Z, finishTime=2025-03-11T08:20:48.269171Z, status=FAILED, progress=null, errorMsg=(ROOTSERVICE)addr: 10.1.46.161:2882, result: -9120(cannot find object on object storage), trace_id: YB420A012EA1-0006300C870CB43A-0-0}] 2025-03-11 16:21:16.970 WARN 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.b.i.t.s.WaitDataBackupSuccessTask : check backup task failed: [DataBackupHistoryWrapperTask{taskId=10, backupType=FULL, clusterId=null, obTenantId=1002, backupSetId=10, backupDest=file:///data/nfs/test/1741079861/tenant_incarnation_1/1002/data, snapshotVersionTime=null, startTime=2025-03-11T08:20:15.562232Z, finishTime=2025-03-11T08:20:48.269171Z, status=FAILED, progress=null, errorMsg=(ROOTSERVICE)addr: 10.1.46.161:2882, result: -9120(cannot find object on object storage), trace_id: YB420A012EA1-0006300C870CB43A-0-0}] 2025-03-11 16:21:16.974 ERROR 113014 --- [manual-subtask-executor15,fc04928ef1c52958,13e012d90a4e69bb] c.o.o.c.t.e.c.w.subtask.SubtaskExecutor : Data backup task failed. com.oceanbase.ocp.core.exception.UnexpectedException: [OCP UnexpectedException]: status=500 INTERNAL_SERVER_ERROR, errorCode=BACKUP_DATA_BACKUP_TASK_FAILED, args= at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.checkDataBackupTaskFinished(WaitDataBackupSuccessTask.java:170) at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.lambda$run$5(WaitDataBackupSuccessTask.java:211) at com.github.rholder.retry.AttemptTimeLimiters$NoAttemptTimeLimit.call(AttemptTimeLimiters.java:78) at com.github.rholder.retry.Retryer.call(Retryer.java:160) at com.oceanbase.ocp.backup.internal.task.schedule.WaitDataBackupSuccessTask.run(WaitDataBackupSuccessTask.java:206) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.execute(JavaSubtaskRunner.java:64) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.doRun(JavaSubtaskRunner.java:32) at com.oceanbase.ocp.core.task.engine.runner.JavaSubtaskRunner.run(JavaSubtaskRunner.java:26) at com.oceanbase.ocp.core.task.engine.runner.RunnerFactory.doRun(RunnerFactory.java:76) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.doRun(SubtaskExecutor.java:207) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.redirectConsoleOutput(SubtaskExecutor.java:201) at com.oceanbase.ocp.core.task.engine.coordinator.worker.subtask.SubtaskExecutor.lambda$submit$2(SubtaskExecutor.java:137) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Set state for subtask: 5556, operation:EXECUTE, state: FAILED