AzCopy panics when uploading 16.5 TB disk snapshot to a container in blob storage

cmd/zc_enumerator.go

@@ -234,6 +234,8 @@ type blobPropsProvider interface {
 	LeaseDuration() lease.DurationType
 	LeaseState() lease.StateType
 	ArchiveStatus() blob.ArchiveStatus
+	LastModified() time.Time
+	ContentLength() int64
 }
 type filePropsProvider interface {
 	contentPropsProvider

cmd/zc_newobjectadapters.go

@@ -62,6 +62,10 @@ func (e emptyPropertiesAdapter) ContentMD5() []byte {
 	return make([]byte, 0)
 }
 
+func (e emptyPropertiesAdapter) ContentLength() int64 {
+	return 0
+}
+
 func (e emptyPropertiesAdapter) BlobType() blob.BlobType {
 	return ""
 }
@@ -74,6 +78,10 @@ func (e emptyPropertiesAdapter) ArchiveStatus() blob.ArchiveStatus {
 	return ""
 }
 
+func (e emptyPropertiesAdapter) LastModified() time.Time {
+	return time.Time{}
+}
+
 func (e emptyPropertiesAdapter) LeaseDuration() lease.DurationType {
 	return ""
 }
@@ -91,6 +99,10 @@ type blobPropertiesResponseAdapter struct {
 	*blob.GetPropertiesResponse
 }
 
+func (a blobPropertiesResponseAdapter) LastModified() time.Time {
+	return common.IffNotNil(a.GetPropertiesResponse.LastModified, time.Time{})
+}
+
 func (a blobPropertiesResponseAdapter) CacheControl() string {
 	return common.IffNotNil(a.GetPropertiesResponse.CacheControl, "")
 }
@@ -115,6 +127,10 @@ func (a blobPropertiesResponseAdapter) ContentMD5() []byte {
 	return a.GetPropertiesResponse.ContentMD5
 }
 
+func (a blobPropertiesResponseAdapter) ContentLength() int64 {
+	return common.IffNotNil(a.GetPropertiesResponse.ContentLength, 0)
+}
+
 func (a blobPropertiesResponseAdapter) BlobType() blob.BlobType {
 	return common.IffNotNil(a.GetPropertiesResponse.BlobType, "")
 }
@@ -172,6 +188,10 @@ func (a blobPropertiesAdapter) ContentMD5() []byte {
 	return a.BlobProperties.ContentMD5
 }
 
+func (a blobPropertiesAdapter) ContentLength() int64 {
+	return common.IffNotNil(a.BlobProperties.ContentLength, 0)
+}
+
 func (a blobPropertiesAdapter) BlobType() blob.BlobType {
 	return common.IffNotNil(a.BlobProperties.BlobType, "")
 }
@@ -199,6 +219,10 @@ func (a blobPropertiesAdapter) ArchiveStatus() blob.ArchiveStatus {
 	return common.IffNotNil(a.BlobProperties.ArchiveStatus, "")
 }
 
+func (a blobPropertiesAdapter) LastModified() time.Time {
+	return common.IffNotNil(a.BlobProperties.LastModified, time.Time{})
+}
+
 type shareFilePropertiesAdapter struct {
 	*sharefile.GetPropertiesResponse
 }
@@ -285,4 +309,4 @@ func (a shareDirectoryPropertiesAdapter) ContentMD5() []byte {
 
 func (a shareDirectoryPropertiesAdapter) ContentLength() int64 {
 	return 0
-}
+}

cmd/zc_traverser_blob.go

@@ -223,16 +223,17 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro
 			azcopyScanningLogger.Log(common.LogDebug, fmt.Sprintf("Root entity type: %s", getEntityType(blobProperties.Metadata)))
 		}
 
+		blobPropsAdapter := blobPropertiesResponseAdapter{blobProperties}
 		storedObject := newStoredObject(
 			preprocessor,
 			getObjectNameOnly(strings.TrimSuffix(blobURLParts.BlobName, common.AZCOPY_PATH_SEPARATOR_STRING)),
 			"",
-			getEntityType(blobProperties.Metadata),
-			*blobProperties.LastModified,
-			*blobProperties.ContentLength,
-			blobPropertiesResponseAdapter{blobProperties},
-			blobPropertiesResponseAdapter{blobProperties},
-			blobProperties.Metadata,
+			getEntityType(blobPropsAdapter.Metadata),
+			blobPropsAdapter.LastModified(),
+			blobPropsAdapter.ContentLength(),
+			blobPropsAdapter,
+			blobPropsAdapter,
+			blobPropsAdapter.Metadata,
 			blobURLParts.ContainerName,
 		)
 
@@ -340,6 +341,7 @@ func (t *blobTraverser) parallelList(containerClient *container.Client, containe
 						// try to get properties on the directory itself, since it's not listed in BlobItems
 						blobClient := containerClient.NewBlobClient(strings.TrimSuffix(*virtualDir.Name, common.AZCOPY_PATH_SEPARATOR_STRING))
 						pResp, err := blobClient.GetProperties(t.ctx, nil)
+						pbPropAdapter := blobPropertiesResponseAdapter{&pResp}
 						folderRelativePath := strings.TrimSuffix(*virtualDir.Name, common.AZCOPY_PATH_SEPARATOR_STRING)
 						folderRelativePath = strings.TrimPrefix(folderRelativePath, searchPrefix)
 						if err == nil {
@@ -348,11 +350,11 @@ func (t *blobTraverser) parallelList(containerClient *container.Client, containe
 								getObjectNameOnly(strings.TrimSuffix(*virtualDir.Name, common.AZCOPY_PATH_SEPARATOR_STRING)),
 								folderRelativePath,
 								common.EEntityType.Folder(),
-								*pResp.LastModified,
-								*pResp.ContentLength,
-								blobPropertiesResponseAdapter{&pResp},
-								blobPropertiesResponseAdapter{&pResp},
-								pResp.Metadata,
+								pbPropAdapter.LastModified(),
+								pbPropAdapter.ContentLength(),
+								pbPropAdapter,
+								pbPropAdapter,
+								pbPropAdapter.Metadata,
 								containerName,
 							)
 
@@ -466,8 +468,8 @@ func (t *blobTraverser) createStoredObjectForBlob(preprocessor objectMorpher, bl
 		getObjectNameOnly(*blobInfo.Name),
 		relativePath,
 		getEntityType(blobInfo.Metadata),
-		*blobInfo.Properties.LastModified,
-		*blobInfo.Properties.ContentLength,
+		adapter.LastModified(),
+		*adapter.BlobProperties.ContentLength,
 		adapter,
 		adapter, // adapter satisfies both interfaces
 		blobInfo.Metadata,

cmd/zc_traverser_blob_versions.go

@@ -87,17 +87,19 @@ func (t *blobVersionsTraverser) Traverse(preprocessor objectMorpher, processor o
 		if blobProperties == nil {
 			panic("isBlob should never be set if getting properties is an error")
 		}
+
+		blobPropsAdapter := blobPropertiesResponseAdapter{blobProperties}
 		blobURLParts.VersionID = versionID
 		storedObject := newStoredObject(
 			preprocessor,
 			getObjectNameOnly(strings.TrimSuffix(blobURLParts.BlobName, common.AZCOPY_PATH_SEPARATOR_STRING)),
 			"",
 			common.EEntityType.File(),
-			*blobProperties.LastModified,
-			*blobProperties.ContentLength,
-			blobPropertiesResponseAdapter{blobProperties},
-			blobPropertiesResponseAdapter{blobProperties},
-			blobProperties.Metadata,
+			blobPropsAdapter.LastModified(),
+			blobPropsAdapter.ContentLength(),
+			blobPropsAdapter,
+			blobPropsAdapter,
+			blobPropsAdapter.Metadata,
 			blobURLParts.ContainerName,
 		)
 		storedObject.blobVersionID = versionID

cmd/zt_traverser_blob_test.go

@@ -24,11 +24,14 @@ import (
 	"context"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	datalakedirectory "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/directory"
 	"github.com/Azure/azure-storage-azcopy/v10/common"
 	"github.com/Azure/azure-storage-azcopy/v10/ste"
 	"github.com/stretchr/testify/assert"
 	"testing"
+	"time"
 )
 
 func TestIsSourceDirWithStub(t *testing.T) {
@@ -289,3 +292,39 @@ func TestGetEntityType(t *testing.T) {
 	a.Equal(common.EEntityType.Symlink(), entityType)
 
 }
+
+func TestBlobPropertiesAdapter_LMTAndContentLength(t *testing.T) {
+	a := assert.New(t)
+
+	// Test case 1: GetPropertiesResponse contains nil ContentLength and LMT
+	props := blobPropertiesResponseAdapter{GetPropertiesResponse: &blob.GetPropertiesResponse{ContentLength: nil, LastModified: nil}}
+
+	a.Equal(props.LastModified(), time.Time{})
+	a.Equal(props.ContentLength(), int64(0))
+
+	// Test case 2: GetPropertiesResponse contains nil ContentLength and LMT
+	length := int64(10)
+	time := time.Now()
+	props2 := blobPropertiesResponseAdapter{GetPropertiesResponse: &blob.GetPropertiesResponse{ContentLength: &length, LastModified: &time}}
+
+	a.Equal(props2.LastModified(), time)
+	a.Equal(props2.ContentLength(), length)
+}
+
+func TestContainerBlobPropertiesAdapter_LMTAndContentLength(t *testing.T) {
+	a := assert.New(t)
+
+	// Test case 1: GetPropertiesResponse contains nil ContentLength and LMT
+	props := blobPropertiesAdapter{BlobProperties: &container.BlobProperties{ContentLength: nil, LastModified: nil}}
+
+	a.Equal(props.LastModified(), time.Time{})
+	a.Equal(props.ContentLength(), int64(0))
+
+	// Test case 2: GetPropertiesResponse contains nil ContentLength and LMT
+	length := int64(10)
+	time := time.Now()
+	props2 := blobPropertiesAdapter{BlobProperties: &container.BlobProperties{ContentLength: &length, LastModified: &time}}
+
+	a.Equal(props2.LastModified(), time)
+	a.Equal(props2.ContentLength(), length)
+}

cmd/zt_traverser_managed_disk_test.go

@@ -0,0 +1,52 @@
+package cmd
+
+import "testing"
+
+func TestLargeManagedDiskSnapshot(t *testing.T) {
+	//a := assert.New(t)
+	//
+	//// Set up for this test.
+	//// In Azure Portal create a managed disk of size 7.9 TB
+	//// Select 'Create Snapshot'
+	//// Under Snapshot Export
+	//// Click on Generate URL and paste that URL below as largeMDSnapshot
+	// largeMDSnapshot := "https://md-XXXXX.blob.storage.azure.net/XXXX/XXXX?snapshot=XXXXXXXXsv=2018-03-28&sr=b&si=XXXXXXX&sig=XXXXXXXXXX"
+	//serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, largeMDSnapshot)
+	//
+	//// Create a page blob client with OAuth
+	//blobClient, err := blob.NewClientWithNoCredential(largeMDSnapshot, nil)
+	//a.NoError(err)
+	//
+	//blobProps, err := blobClient.GetProperties(context.TODO(), nil)
+	//a.NoError(err)
+	//a.Nil(blobProps.LastModified)
+	//
+	//blobTraverser := newBlobTraverser(largeMDSnapshot, serviceClientWithSAS, ctx, true, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false)
+	//blobDummyProcessor := dummyProcessor{}
+	//err = blobTraverser.Traverse(noPreProccessor, blobDummyProcessor.process, nil)
+	//a.NoError(err)
+}
+
+func TestLargeManagedDisk(t *testing.T) {
+	//a := assert.New(t)
+	//
+	//// Set up for this test.
+	//// In Azure Portal create a managed disk of size 7.9 TB
+	//// Under Disk Export
+	//// Click on Generate URL and paste that URL below as largeMD
+	//// largeMD := "https://md-XXXXX.blob.storage.azure.net/XXXX/XXXX?sv=2018-03-28&sr=b&si=XXXXXXX&sig=XXXXXXXXXX"
+	//serviceClientWithSAS := scenarioHelper{}.getBlobServiceClientWithSASFromURL(a, largeMD)
+	//
+	//// Create a page blob client with OAuth
+	//blobClient, err := blob.NewClientWithNoCredential(largeMD, nil)
+	//a.NoError(err)
+	//
+	//blobProps, err := blobClient.GetProperties(context.TODO(), nil)
+	//a.NoError(err)
+	//a.Nil(blobProps.LastModified)
+	//
+	//blobTraverser := newBlobTraverser(largeMD, serviceClientWithSAS, ctx, true, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None(), false)
+	//blobDummyProcessor := dummyProcessor{}
+	//err = blobTraverser.Traverse(noPreProccessor, blobDummyProcessor.process, nil)
+	//a.NoError(err)
+}

e2etest/config.go

@@ -123,13 +123,16 @@ func (AccountType) GCP() AccountType                          { return AccountTy
 func (AccountType) Azurite() AccountType                      { return AccountType(8) }
 func (AccountType) ManagedDiskSnapshot() AccountType          { return AccountType(9) }
 func (AccountType) ManagedDiskSnapshotOAuth() AccountType     { return AccountType(10) }
+func (AccountType) LargeManagedDiskSnapshot() AccountType     { return AccountType(11) }
+func (AccountType) LargeManagedDisk() AccountType             { return AccountType(12) }
 
 func (o AccountType) String() string {
 	return enum.StringInt(o, reflect.TypeOf(o))
 }
 
 func (o AccountType) IsManagedDisk() bool {
-	return o == o.StdManagedDisk() || o == o.OAuthManagedDisk() || o == o.ManagedDiskSnapshot() || o == o.ManagedDiskSnapshotOAuth()
+	return o == o.StdManagedDisk() || o == o.OAuthManagedDisk() || o == o.ManagedDiskSnapshot() || o == o.ManagedDiskSnapshotOAuth() ||
+		o == o.LargeManagedDiskSnapshot() || o == o.LargeManagedDisk()
 }
 
 func (o AccountType) IsBlobOnly() bool {
@@ -144,7 +147,7 @@ type ManagedDiskConfig struct {
 	ResourceGroupName string
 	DiskName          string
 
-	oauth             AccessToken
+	oauth      AccessToken
 	isSnapshot bool
 }
 
@@ -159,12 +162,17 @@ func (gim GlobalInputManager) GetMDConfig(accountType AccountType) (*ManagedDisk
 		mdConfigVar = "AZCOPY_E2E_STD_MANAGED_DISK_CONFIG"
 	case EAccountType.OAuthManagedDisk():
 		mdConfigVar = "AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG"
+	case EAccountType.LargeManagedDisk():
+		mdConfigVar = "AZCOPY_E2E_LARGE_MANAGED_DISK_CONFIG"
 	case EAccountType.ManagedDiskSnapshot():
 		mdConfigVar = "AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG"
 		isSnapshot = true
 	case EAccountType.ManagedDiskSnapshotOAuth():
 		mdConfigVar = "AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG"
 		isSnapshot = true
+	case EAccountType.LargeManagedDiskSnapshot():
+		mdConfigVar = "AZCOPY_E2E_LARGE_MANAGED_DISK_SNAPSHOT_CONFIG"
+		isSnapshot = true
 	default:
 		return nil, fmt.Errorf("account type %s is invalid for GetMDConfig", accountType.String())
 	}

e2etest/declarativeScenario.go

@@ -113,8 +113,10 @@ func (s *scenario) Run() {
 	if s.destAccountType.IsManagedDisk() {
 		s.a.Assert(s.destAccountType, notEquals(), EAccountType.StdManagedDisk(), "Upload is not supported in MD testing yet")
 		s.a.Assert(s.destAccountType, notEquals(), EAccountType.OAuthManagedDisk(), "Upload is not supported in MD testing yet")
+		s.a.Assert(s.destAccountType, notEquals(), EAccountType.LargeManagedDisk(), "Upload is not supported in MD testing yet")
 		s.a.Assert(s.destAccountType, notEquals(), EAccountType.ManagedDiskSnapshot(), "Cannot upload to a MD snapshot")
 		s.a.Assert(s.destAccountType, notEquals(), EAccountType.ManagedDiskSnapshotOAuth(), "Cannot upload to a MD snapshot")
+		s.a.Assert(s.destAccountType, notEquals(), EAccountType.LargeManagedDiskSnapshot(), "Cannot upload to a MD snapshot")
 		s.a.Assert(true, equals(), s.fromTo.From() == common.ELocation.Blob() || s.fromTo.From() == common.ELocation.BlobFS())
 	}
 

e2etest/helpers.go

@@ -27,6 +27,7 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/binary"
+	"flag"
 	"fmt"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
@@ -78,6 +79,8 @@ const (
 	defaultBlobFSFileSizeInBytes = 1000
 )
 
+var runLocallyOnly = flag.Bool("local-tests", false, "Tests with this flag are run locally only")
+
 func pointerTo[T any](in T) *T {
 	return &in
 }