Failed to read snapshot's definition "snapshot-1859": ServerFaultCode

FULL LOG

2019-08-05T09:54:32Z NOTICE: [74] Logging to '/opt/emc/vproxy/runtime/logs/vbackupd/BackupVmSessions-a65dd848-c7cd-44a3-8511-69650dfb8ed6.log' on host 'dell-vproxy1.metaljunction.com'.
2019-08-05T09:54:32Z NOTICE: [74] Release: '4.0.0-7_1', Build number: '1', Build date: '2019-04-18T03:58:31Z'
2019-08-05T09:54:32Z NOTICE: [74] Changing master log level from INFO to TRACE.
2019-08-05T09:54:32Z NOTICE: [74] Changing master log level from TRACE to INFO.
2019-08-05T09:54:32Z NOTICE: [74] Setting max concurrency to 15.
2019-08-05T09:54:32Z NOTICE: [74] Set buffer count to 2.
2019-08-05T09:54:32Z NOTICE: [74] Set read buffer size to 10 MiB.
2019-08-05T09:54:32Z NOTICE: [74] Starting backup of "DELL-ENT-MGR" (vm-1144) logging at level "INFO" ...
2019-08-05T09:54:32Z NOTICE: [74] BackupVmSessions supplied by client:
2019-08-05T09:54:32Z NOTICE: [74] {
2019-08-05T09:54:32Z NOTICE: [74] "Config": {
2019-08-05T09:54:32Z NOTICE: [74] "SessionId": "a65dd848-c7cd-44a3-8511-69650dfb8ed6",
2019-08-05T09:54:32Z NOTICE: [74] "LogTag": "74",
2019-08-05T09:54:32Z NOTICE: [74] "LogLevel": "INFO",
2019-08-05T09:54:32Z NOTICE: [74] "IdleTimeout": 300,
2019-08-05T09:54:32Z NOTICE: [74] "AcceptedLanguage": "en",
2019-08-05T09:54:32Z NOTICE: [74] "HypervisorType": "",
2019-08-05T09:54:32Z NOTICE: [74] "TransportModeRequested": "hotadd:nbd",
2019-08-05T09:54:32Z NOTICE: [74] "Parallelism": 1,
2019-08-05T09:54:32Z NOTICE: [74] "UseCbt": true,
2019-08-05T09:54:32Z NOTICE: [74] "VimServerRef": {
2019-08-05T09:54:32Z NOTICE: [74] "HostName": "172.16.1.61",
2019-08-05T09:54:32Z NOTICE: [74] "TcpPort": 0,
2019-08-05T09:54:32Z NOTICE: [74] "UserName": "dellemcvproxy",
2019-08-05T09:54:32Z NOTICE: [74] "UserPassword": "********",
2019-08-05T09:54:32Z NOTICE: [74] "ThumbPrint": "",
2019-08-05T09:54:32Z NOTICE: [74] "HypervisorMode": "vSphere"
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "VmSpec": {
2019-08-05T09:54:32Z NOTICE: [74] "Name": "DELL-ENT-MGR",
2019-08-05T09:54:32Z NOTICE: [74] "VmMoref": "vm-1144",
2019-08-05T09:54:32Z NOTICE: [74] "VirtualDisks": [
2019-08-05T09:54:32Z NOTICE: [74] {
2019-08-05T09:54:32Z NOTICE: [74] "Label": "Hard disk 1",
2019-08-05T09:54:32Z NOTICE: [74] "Key": 2000
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] {
2019-08-05T09:54:32Z NOTICE: [74] "Label": "Hard disk 2",
2019-08-05T09:54:32Z NOTICE: [74] "Key": 2001
2019-08-05T09:54:32Z NOTICE: [74] }
2019-08-05T09:54:32Z NOTICE: [74] ],
2019-08-05T09:54:32Z NOTICE: [74] "CustomFieldName": "Last EMC vProxy Backup",
2019-08-05T09:54:32Z NOTICE: [74] "CustomFieldValue": "Backup Server=dell-emcmgr-1, Policy=Gold, Workflow=Test-Vm backup, Action=backup, JobId=97291"
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "SnapshotSpec": {
2019-08-05T09:54:32Z NOTICE: [74] "Name": "NetWorker Backup Snapshot",
2019-08-05T09:54:32Z NOTICE: [74] "Description": "Snapshot is created as part of protecting this VM by EMC NetWorker VProxy.",
2019-08-05T09:54:32Z NOTICE: [74] "Quiesce": true,
2019-08-05T09:54:32Z NOTICE: [74] "QuiesceTimeout": 30,
2019-08-05T09:54:32Z NOTICE: [74] "QuiesceType": "COPY",
2019-08-05T09:54:32Z NOTICE: [74] "DumpMemory": false,
2019-08-05T09:54:32Z NOTICE: [74] "RemoveAll": false,
2019-08-05T09:54:32Z NOTICE: [74] "Consolidate": false
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "TargetSpec": {
2019-08-05T09:54:32Z NOTICE: [74] "DeviceType": "DataDomain",
2019-08-05T09:54:32Z NOTICE: [74] "HostName": "mj-kol-dd6300.metaljunction.com",
2019-08-05T09:54:32Z NOTICE: [74] "UserName": "ddboost",
2019-08-05T09:54:32Z NOTICE: [74] "UserPassword": "********",
2019-08-05T09:54:32Z NOTICE: [74] "BackupMode": "VSS",
2019-08-05T09:54:32Z NOTICE: [74] "BackupPath": "dell-emcmgr-1/Networkar-backup//dell-emcmgr-1/Networkar-backup/95/58/73d4322e-00000006-de47fcd8-5d47fcd8-00235000-163dbf56",
2019-08-05T09:54:32Z NOTICE: [74] "EncryptOnWire": false,
2019-08-05T09:54:32Z NOTICE: [74] "PreviousBackupPath": "",
2019-08-05T09:54:32Z NOTICE: [74] "CurrentBackupPath": "/dell-emcmgr-1/Networkar-backup/active/73d4322e-00000006-de47fcd8-5d47fcd8-00235000-163dbf56",
2019-08-05T09:54:32Z NOTICE: [74] "BackupRoot": "",
2019-08-05T09:54:32Z NOTICE: [74] "TargetTag": "",
2019-08-05T09:54:32Z NOTICE: [74] "ForeignBackup": {
2019-08-05T09:54:32Z NOTICE: [74] "Type": "",
2019-08-05T09:54:32Z NOTICE: [74] "Path": ""
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "VBABackup": {
2019-08-05T09:54:32Z NOTICE: [74] "VBABackupPath": "",
2019-08-05T09:54:32Z NOTICE: [74] "VBATempPath": ""
2019-08-05T09:54:32Z NOTICE: [74] }
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "GuestSpec": {
2019-08-05T09:54:32Z NOTICE: [74] "VmGuestInfo": {
2019-08-05T09:54:32Z NOTICE: [74] "VmName": "",
2019-08-05T09:54:32Z NOTICE: [74] "Hostname": "",
2019-08-05T09:54:32Z NOTICE: [74] "MorefId": "",
2019-08-05T09:54:32Z NOTICE: [74] "UserName": "",
2019-08-05T09:54:32Z NOTICE: [74] "UserPassword": "",
2019-08-05T09:54:32Z NOTICE: [74] "InstallationAuthorized": false,
2019-08-05T09:54:32Z NOTICE: [74] "RemoveInstallation": false,
2019-08-05T09:54:32Z NOTICE: [74] "AdminUser": "",
2019-08-05T09:54:32Z NOTICE: [74] "AdminPassword": ""
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "AppName": "",
2019-08-05T09:54:32Z NOTICE: [74] "BackupType": "FULL",
2019-08-05T09:54:32Z NOTICE: [74] "ApplicationData": {
2019-08-05T09:54:32Z NOTICE: [74] "BackupTime": "Type=string",
2019-08-05T09:54:32Z NOTICE: [74] "Mode": "Type=string"
2019-08-05T09:54:32Z NOTICE: [74] }
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "ZeroFillUnusedSectors": false,
2019-08-05T09:54:32Z NOTICE: [74] "AutoEnableCbt": true,
2019-08-05T09:54:32Z NOTICE: [74] "AutoRepairCbt": true,
2019-08-05T09:54:32Z NOTICE: [74] "FallbackCbtThreshold": 0,
2019-08-05T09:54:32Z NOTICE: [74] "FallbackTransportMode": "",
2019-08-05T09:54:32Z NOTICE: [74] "DeprecatedREST": false,
2019-08-05T09:54:32Z NOTICE: [74] "ReadBufferMB": 10,
2019-08-05T09:54:32Z NOTICE: [74] "WriteBufferKB": 0,
2019-08-05T09:54:32Z NOTICE: [74] "BufferCount": 2,
2019-08-05T09:54:32Z NOTICE: [74] "BackupId": "",
2019-08-05T09:54:32Z NOTICE: [74] "BackupTime": "0001-01-01T00:00:00Z"
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "Actions": {
2019-08-05T09:54:32Z NOTICE: [74] "ChangeSession": "",
2019-08-05T09:54:32Z NOTICE: [74] "DeleteSession": "/api/v1/BackupVmSessions/a65dd848-c7cd-44a3-8511-69650dfb8ed6",
2019-08-05T09:54:32Z NOTICE: [74] "CancelActivity": "/api/v1/BackupVmSessions/a65dd848-c7cd-44a3-8511-69650dfb8ed6/activity",
2019-08-05T09:54:32Z NOTICE: [74] "GetSession": "/api/v1/BackupVmSessions/a65dd848-c7cd-44a3-8511-69650dfb8ed6",
2019-08-05T09:54:32Z NOTICE: [74] "GetSessionLog": "/api/v1/BackupVmSessions/a65dd848-c7cd-44a3-8511-69650dfb8ed6/log",
2019-08-05T09:54:32Z NOTICE: [74] "GetAllSessions": "/api/v1/BackupVmSessions"
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "Status": {
2019-08-05T09:54:32Z NOTICE: [74] "State": "Queued",
2019-08-05T09:54:32Z NOTICE: [74] "Description": "",
2019-08-05T09:54:32Z NOTICE: [74] "Progress": 0,
2019-08-05T09:54:32Z NOTICE: [74] "Error": {
2019-08-05T09:54:32Z NOTICE: [74] "Text": "",
2019-08-05T09:54:32Z NOTICE: [74] "Code": 0,
2019-08-05T09:54:32Z NOTICE: [74] "Classifications": null,
2019-08-05T09:54:32Z NOTICE: [74] "Retryable": false,
2019-08-05T09:54:32Z NOTICE: [74] "Remedy": null
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "Warnings": null,
2019-08-05T09:54:32Z NOTICE: [74] "StartTime": "",
2019-08-05T09:54:32Z NOTICE: [74] "EndTime": "",
2019-08-05T09:54:32Z NOTICE: [74] "ApplicationData": "",
2019-08-05T09:54:32Z NOTICE: [74] "ApplicationVersion": "",
2019-08-05T09:54:32Z NOTICE: [74] "BytesToMove": 0,
2019-08-05T09:54:32Z NOTICE: [74] "BytesMoved": 0,
2019-08-05T09:54:32Z NOTICE: [74] "TransportModeUsed": "",
2019-08-05T09:54:32Z NOTICE: [74] "BackupModeUsed": "",
2019-08-05T09:54:32Z NOTICE: [74] "VirtualDiskStatus": null,
2019-08-05T09:54:32Z NOTICE: [74] "AppStatus": {
2019-08-05T09:54:32Z NOTICE: [74] "Progress": 0,
2019-08-05T09:54:32Z NOTICE: [74] "TotalBytesToProcess": 0,
2019-08-05T09:54:32Z NOTICE: [74] "TotalBytesProcessed": 0,
2019-08-05T09:54:32Z NOTICE: [74] "TotalObjectsToProcess": 0,
2019-08-05T09:54:32Z NOTICE: [74] "TotalObjectsProcessed": 0,
2019-08-05T09:54:32Z NOTICE: [74] "ObjectType": "",
2019-08-05T09:54:32Z NOTICE: [74] "Custom": null
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "Manifest": {
2019-08-05T09:54:32Z NOTICE: [74] "BackupFilePaths": null,
2019-08-05T09:54:32Z NOTICE: [74] "AppAgentBackup": {
2019-08-05T09:54:32Z NOTICE: [74] "AppName": "",
2019-08-05T09:54:32Z NOTICE: [74] "AppBackupType": "",
2019-08-05T09:54:32Z NOTICE: [74] "AppMetaValues": null
2019-08-05T09:54:32Z NOTICE: [74] }
2019-08-05T09:54:32Z NOTICE: [74] },
2019-08-05T09:54:32Z NOTICE: [74] "BackupLevelPerformed": "",
2019-08-05T09:54:32Z NOTICE: [74] "SnapshotStatus": {
2019-08-05T09:54:32Z NOTICE: [74] "Quiesced": false,
2019-08-05T09:54:32Z NOTICE: [74] "QuiesceType": ""
2019-08-05T09:54:32Z NOTICE: [74] }
2019-08-05T09:54:32Z NOTICE: [74] }
2019-08-05T09:54:32Z NOTICE: [74] }
2019-08-05T09:54:32Z NOTICE: [74] DD Host = 'mj-kol-dd6300.metaljunction.com', DD Model = 'DD6300', DDOS Version = 'Data Domain OS 6.1.2.20-606786', DDBoost Version = '3.5.0.0-607663'
2019-08-05T09:54:32Z INFO: [74] DDOS version 6.1.2.20-606786 is compatible.
2019-08-05T09:54:32Z INFO: [74] Connected to session on vCenter '172.16.1.61' Version '5.1.0 build-799731, API:5.1', Uuid:0BD6E49F-BCF5-4F24-BD47-17E4EB4407DD.
2019-08-05T09:54:32Z INFO: [74] Found virtual machine 'DELL-ENT-MGR' with MORef 'vm-1144'.
2019-08-05T09:54:33Z INFO: [74] Host '172.16.1.67' is running VMware ESXi 5.1.0 build-799733.
2019-08-05T09:54:33Z INFO: [74] Locking VM.
2019-08-05T09:54:33Z INFO: [74] Set custom attribute 'Dell EMC vProxy Session' value for object vm-1144 to 'Operation=Backup, StartTime=2019-08-05T09:54:32Z, vProxy=dell-vproxy1.metaljunction.com, SessionId=a65dd848-c7cd-44a3-8511-69650dfb8ed6'.
2019-08-05T09:54:46Z NOTICE: [74] Successfully removed snapshot "snapshot-1860".
2019-08-05T09:54:46Z INFO: [74] Disabling storage migration of virtual machine ...
2019-08-05T09:54:48Z NOTICE: [74] Storage migration successfully disabled: "vm-1144" @ "172.16.1.61"
2019-08-05T09:54:48Z INFO: [74] Creating snapshot of virtual machine ...
2019-08-05T09:55:35Z INFO: [74] Create snapshot request succeeded.
2019-08-05T09:55:35Z ERROR: [74] Failed to read snapshot's definition "snapshot-1861": ServerFaultCode:
2019-08-05T09:55:35Z ERROR: [74] Unable to find snapshot "snapshot-1861" of virtual machine: ServerFaultCode:
2019-08-05T09:55:35Z INFO: [74] Enabling storage migration of virtual machine ...
2019-08-05T09:55:36Z INFO: [74] Storage migration successfully enabled: "vm-1144" @ "172.16.1.61"
2019-08-05T09:55:36Z INFO: [74] Storage migration enabled.
2019-08-05T09:55:36Z INFO: [74] Set custom attribute 'Dell EMC vProxy Session' value for object vm-1144 to ''.
2019-08-05T09:55:36Z INFO: [74] Unlocked virtual machine.
2019-08-05T09:55:36Z INFO: [74] vProxy locks reset successfully.
2019-08-05T09:55:36Z INFO: [74] Set custom attribute 'Dell EMC vProxy Cleanup Descriptor' value for object vm-1144 to ''.
2019-08-05T09:55:36Z INFO: [74] Disconnected from session on vCenter '172.16.1.61'.
2019-08-05T09:55:36Z ERROR: [74] Backup has FAILED.
2019-08-05T09:55:36Z NOTICE: [74] Final status of session:
2019-08-05T09:55:36Z NOTICE: [74] {
2019-08-05T09:55:36Z NOTICE: [74] "Config": {
2019-08-05T09:55:36Z NOTICE: [74] "SessionId": "a65dd848-c7cd-44a3-8511-69650dfb8ed6",
2019-08-05T09:55:36Z NOTICE: [74] "LogTag": "74",
2019-08-05T09:55:36Z NOTICE: [74] "LogLevel": "INFO",
2019-08-05T09:55:36Z NOTICE: [74] "IdleTimeout": 300,
2019-08-05T09:55:36Z NOTICE: [74] "AcceptedLanguage": "en",
2019-08-05T09:55:36Z NOTICE: [74] "HypervisorType": "",
2019-08-05T09:55:36Z NOTICE: [74] "TransportModeRequested": "hotadd:nbd",
2019-08-05T09:55:36Z NOTICE: [74] "Parallelism": 1,
2019-08-05T09:55:36Z NOTICE: [74] "UseCbt": true,
2019-08-05T09:55:36Z NOTICE: [74] "VimServerRef": {
2019-08-05T09:55:36Z NOTICE: [74] "HostName": "172.16.1.61",
2019-08-05T09:55:36Z NOTICE: [74] "TcpPort": 0,
2019-08-05T09:55:36Z NOTICE: [74] "UserName": "dellemcvproxy",
2019-08-05T09:55:36Z NOTICE: [74] "UserPassword": "********",
2019-08-05T09:55:36Z NOTICE: [74] "ThumbPrint": "",
2019-08-05T09:55:36Z NOTICE: [74] "HypervisorMode": "vSphere"
2019-08-05T09:55:36Z NOTICE: [74] },
2019-08-05T09:55:36Z NOTICE: [74] "VmSpec": {
2019-08-05T09:55:36Z NOTICE: [74] "Name": "DELL-ENT-MGR",
2019-08-05T09:55:36Z NOTICE: [74] "VmMoref": "vm-1144",
2019-08-05T09:55:36Z NOTICE: [74] "VirtualDisks": [
2019-08-05T09:55:36Z NOTICE: [74] {
2019-08-05T09:55:36Z NOTICE: [74] "Label": "Hard disk 1",
2019-08-05T09:55:36Z NOTICE: [74] "Key": 2000
2019-08-05T09:55:36Z NOTICE: [74] },
2019-08-05T09:55:36Z NOTICE: [74] {
2019-08-05T09:55:36Z NOTICE: [74] "Label": "Hard disk 2",
2019-08-05T09:55:36Z NOTICE: [74] "Key": 2001
2019-08-05T09:55:36Z NOTICE: [74] }
2019-08-05T09:55:36Z NOTICE: [74] ],
2019-08-05T09:55:36Z NOTICE: [74] "CustomFieldName": "Last EMC vProxy Backup",
2019-08-05T09:55:36Z NOTICE: [74] "

Honestly - what do you expect with this minimalistic kind of info?

Some questions from my side,

- Has this backup worked before or is this a new configuration?

- Any special configuration on this VM? e'g : IDE disk type, RAW disk attached

- Have you enabled quince option?

- Can you try and create a snapshot of the respective VM on vCenter?