AzCopy panics when uploading 16.5 TB disk snapshot to a container in blob storage

azure-pipelines.yml

@@ -219,6 +219,8 @@ jobs:
           AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG)
           AZCOPY_E2E_STD_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_CONFIG)
           AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG)
+          AZCOPY_E2E_LARGE_MANAGED_DISK_CONFIG: $(AZCOPY_E2E_LARGE_MANAGED_DISK_CONFIG)
+          AZCOPY_E2E_LARGE_MANAGED_DISK_SNAPSHOT_CONFIG: $(AZCOPY_E2E_LARGE_MANAGED_DISK_SNAPSHOT_CONFIG)
           CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
           CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
           AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/$(build_name)

cmd/zc_enumerator.go

@@ -234,6 +234,7 @@ type blobPropsProvider interface {
 	LeaseDuration() lease.DurationType
 	LeaseState() lease.StateType
 	ArchiveStatus() blob.ArchiveStatus
+	LastModified() time.Time
 }
 type filePropsProvider interface {
 	contentPropsProvider

cmd/zc_newobjectadapters.go

@@ -74,6 +74,10 @@ func (e emptyPropertiesAdapter) ArchiveStatus() blob.ArchiveStatus {
 	return ""
 }
 
+func (e emptyPropertiesAdapter) LastModified() time.Time {
+	return time.Time{}
+}
+
 func (e emptyPropertiesAdapter) LeaseDuration() lease.DurationType {
 	return ""
 }
@@ -91,6 +95,10 @@ type blobPropertiesResponseAdapter struct {
 	*blob.GetPropertiesResponse
 }
 
+func (a blobPropertiesResponseAdapter) LastModified() time.Time {
+	return common.IffNotNil(a.GetPropertiesResponse.LastModified, time.Time{})
+}
+
 func (a blobPropertiesResponseAdapter) CacheControl() string {
 	return common.IffNotNil(a.GetPropertiesResponse.CacheControl, "")
 }
@@ -199,6 +207,10 @@ func (a blobPropertiesAdapter) ArchiveStatus() blob.ArchiveStatus {
 	return common.IffNotNil(a.BlobProperties.ArchiveStatus, "")
 }
 
+func (a blobPropertiesAdapter) LastModified() time.Time {
+	return common.IffNotNil(a.BlobProperties.LastModified, time.Time{})
+}
+
 type shareFilePropertiesAdapter struct {
 	*sharefile.GetPropertiesResponse
 }
@@ -285,4 +297,4 @@ func (a shareDirectoryPropertiesAdapter) ContentMD5() []byte {
 
 func (a shareDirectoryPropertiesAdapter) ContentLength() int64 {
 	return 0
-}
+}

cmd/zc_traverser_blob.go

@@ -223,16 +223,17 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro
 			azcopyScanningLogger.Log(common.LogDebug, fmt.Sprintf("Root entity type: %s", getEntityType(blobProperties.Metadata)))
 		}
 
+		blobPropsAdapter := blobPropertiesResponseAdapter{blobProperties}
 		storedObject := newStoredObject(
 			preprocessor,
 			getObjectNameOnly(strings.TrimSuffix(blobURLParts.BlobName, common.AZCOPY_PATH_SEPARATOR_STRING)),
 			"",
-			getEntityType(blobProperties.Metadata),
-			*blobProperties.LastModified,
-			*blobProperties.ContentLength,
-			blobPropertiesResponseAdapter{blobProperties},
-			blobPropertiesResponseAdapter{blobProperties},
-			blobProperties.Metadata,
+			getEntityType(blobPropsAdapter.Metadata),
+			blobPropsAdapter.LastModified(),
+			*blobPropsAdapter.ContentLength,
+			blobPropsAdapter,
+			blobPropsAdapter,
+			blobPropsAdapter.Metadata,
 			blobURLParts.ContainerName,
 		)
 
@@ -340,6 +341,7 @@ func (t *blobTraverser) parallelList(containerClient *container.Client, containe
 						// try to get properties on the directory itself, since it's not listed in BlobItems
 						blobClient := containerClient.NewBlobClient(strings.TrimSuffix(*virtualDir.Name, common.AZCOPY_PATH_SEPARATOR_STRING))
 						pResp, err := blobClient.GetProperties(t.ctx, nil)
+						pbPropAdapter := blobPropertiesResponseAdapter{&pResp}
 						folderRelativePath := strings.TrimSuffix(*virtualDir.Name, common.AZCOPY_PATH_SEPARATOR_STRING)
 						folderRelativePath = strings.TrimPrefix(folderRelativePath, searchPrefix)
 						if err == nil {
@@ -348,11 +350,11 @@ func (t *blobTraverser) parallelList(containerClient *container.Client, containe
 								getObjectNameOnly(strings.TrimSuffix(*virtualDir.Name, common.AZCOPY_PATH_SEPARATOR_STRING)),
 								folderRelativePath,
 								common.EEntityType.Folder(),
-								*pResp.LastModified,
-								*pResp.ContentLength,
-								blobPropertiesResponseAdapter{&pResp},
-								blobPropertiesResponseAdapter{&pResp},
-								pResp.Metadata,
+								pbPropAdapter.LastModified(),
+								*pbPropAdapter.ContentLength,
+								pbPropAdapter,
+								pbPropAdapter,
+								pbPropAdapter.Metadata,
 								containerName,
 							)
 
@@ -466,8 +468,8 @@ func (t *blobTraverser) createStoredObjectForBlob(preprocessor objectMorpher, bl
 		getObjectNameOnly(*blobInfo.Name),
 		relativePath,
 		getEntityType(blobInfo.Metadata),
-		*blobInfo.Properties.LastModified,
-		*blobInfo.Properties.ContentLength,
+		adapter.LastModified(),
+		*adapter.BlobProperties.ContentLength,
 		adapter,
 		adapter, // adapter satisfies both interfaces
 		blobInfo.Metadata,

cmd/zc_traverser_blob_versions.go

@@ -87,17 +87,19 @@ func (t *blobVersionsTraverser) Traverse(preprocessor objectMorpher, processor o
 		if blobProperties == nil {
 			panic("isBlob should never be set if getting properties is an error")
 		}
+
+		blobPropsAdapter := blobPropertiesResponseAdapter{blobProperties}
 		blobURLParts.VersionID = versionID
 		storedObject := newStoredObject(
 			preprocessor,
 			getObjectNameOnly(strings.TrimSuffix(blobURLParts.BlobName, common.AZCOPY_PATH_SEPARATOR_STRING)),
 			"",
 			common.EEntityType.File(),
-			*blobProperties.LastModified,
-			*blobProperties.ContentLength,
-			blobPropertiesResponseAdapter{blobProperties},
-			blobPropertiesResponseAdapter{blobProperties},
-			blobProperties.Metadata,
+			blobPropsAdapter.LastModified(),
+			*blobPropsAdapter.ContentLength,
+			blobPropsAdapter,
+			blobPropsAdapter,
+			blobPropsAdapter.Metadata,
 			blobURLParts.ContainerName,
 		)
 		storedObject.blobVersionID = versionID

e2etest/config.go

@@ -123,13 +123,16 @@ func (AccountType) GCP() AccountType                          { return AccountTy
 func (AccountType) Azurite() AccountType                      { return AccountType(8) }
 func (AccountType) ManagedDiskSnapshot() AccountType          { return AccountType(9) }
 func (AccountType) ManagedDiskSnapshotOAuth() AccountType     { return AccountType(10) }
+func (AccountType) LargeManagedDiskSnapshot() AccountType     { return AccountType((11)) }
+func (AccountType) LargeManagedDisk() AccountType             { return AccountType((12)) }
 
 func (o AccountType) String() string {
 	return enum.StringInt(o, reflect.TypeOf(o))
 }
 
 func (o AccountType) IsManagedDisk() bool {
-	return o == o.StdManagedDisk() || o == o.OAuthManagedDisk() || o == o.ManagedDiskSnapshot() || o == o.ManagedDiskSnapshotOAuth()
+	return o == o.StdManagedDisk() || o == o.OAuthManagedDisk() || o == o.ManagedDiskSnapshot() || o == o.ManagedDiskSnapshotOAuth() ||
+		o == o.LargeManagedDiskSnapshot() || o == o.LargeManagedDisk()
 }
 
 func (o AccountType) IsBlobOnly() bool {
@@ -144,7 +147,7 @@ type ManagedDiskConfig struct {
 	ResourceGroupName string
 	DiskName          string
 
-	oauth             AccessToken
+	oauth      AccessToken
 	isSnapshot bool
 }
 
@@ -159,12 +162,17 @@ func (gim GlobalInputManager) GetMDConfig(accountType AccountType) (*ManagedDisk
 		mdConfigVar = "AZCOPY_E2E_STD_MANAGED_DISK_CONFIG"
 	case EAccountType.OAuthManagedDisk():
 		mdConfigVar = "AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG"
+	case EAccountType.LargeManagedDisk():
+		mdConfigVar = "AZCOPY_E2E_LARGE_MANAGED_DISK_CONFIG"
 	case EAccountType.ManagedDiskSnapshot():
 		mdConfigVar = "AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG"
 		isSnapshot = true
 	case EAccountType.ManagedDiskSnapshotOAuth():
 		mdConfigVar = "AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG"
 		isSnapshot = true
+	case EAccountType.LargeManagedDiskSnapshot():
+		mdConfigVar = "AZCOPY_E2E_LARGE_MANAGED_DISK_SNAPSHOT_CONFIG"
+		isSnapshot = true
 	default:
 		return nil, fmt.Errorf("account type %s is invalid for GetMDConfig", accountType.String())
 	}

e2etest/zt_managed_disks_test.go

@@ -147,3 +147,61 @@ func TestManagedDisks_OAuthRequired(t *testing.T) {
 		"",
 	)
 }
+
+// Test a managed disk of size 7.9 TB
+func TestManagedDisks_LargeManagedDisk(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("Limit runs to Linux so no simultaneous runs occur")
+		return
+	}
+
+	RunScenarios(
+		t,
+		eOperation.Copy(),
+		eTestFromTo.Other(common.EFromTo.BlobLocal(), common.EFromTo.BlobLocal()),
+		eValidate.Auto(),
+		anonymousAuthOnly,
+		anonymousAuthOnly,
+		params{
+			disableParallelTesting: true,
+		},
+		nil,
+		testFiles{
+			shouldTransfer: []interface{}{
+				"", // Managed disks will always have a transfer target of ""
+			},
+		},
+		EAccountType.Standard(),
+		EAccountType.StdManagedDisk(),
+		"",
+	)
+}
+
+// Test a managed disk snapshot of size 7.9 TB
+func TestManagedDisks_LargeSnapshot(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("Limit runs to Linux so no simultaneous runs occur")
+		return
+	}
+
+	RunScenarios(
+		t,
+		eOperation.Copy(),
+		eTestFromTo.Other(common.EFromTo.BlobLocal(), common.EFromTo.BlobBlob()),
+		eValidate.Auto(),
+		anonymousAuthOnly,
+		anonymousAuthOnly,
+		params{
+			disableParallelTesting: true,
+		},
+		nil,
+		testFiles{
+			shouldTransfer: []interface{}{
+				"", // Managed disks will always have a transfer target of ""
+			},
+		},
+		EAccountType.Standard(),
+		EAccountType.LargeManagedDiskSnapshot(),
+		"",
+	)
+}