AzCopy panics when uploading 16.5 TB disk snapshot to a container in blob storage

cmd/zc_enumerator.go

@@ -234,6 +234,8 @@ type blobPropsProvider interface {
 	LeaseDuration() lease.DurationType
 	LeaseState() lease.StateType
 	ArchiveStatus() blob.ArchiveStatus
+	LastModified() time.Time
+	ContentLength() int64
 }
 type filePropsProvider interface {
 	contentPropsProvider

cmd/zc_newobjectadapters.go

@@ -62,6 +62,10 @@ func (e emptyPropertiesAdapter) ContentMD5() []byte {
 	return make([]byte, 0)
 }
 
+func (e emptyPropertiesAdapter) ContentLength() int64 {
+	return 0
+}
+
 func (e emptyPropertiesAdapter) BlobType() blob.BlobType {
 	return ""
 }
@@ -74,6 +78,10 @@ func (e emptyPropertiesAdapter) ArchiveStatus() blob.ArchiveStatus {
 	return ""
 }
 
+func (e emptyPropertiesAdapter) LastModified() time.Time {
+	return time.Time{}
+}
+
 func (e emptyPropertiesAdapter) LeaseDuration() lease.DurationType {
 	return ""
 }
@@ -91,6 +99,10 @@ type blobPropertiesResponseAdapter struct {
 	*blob.GetPropertiesResponse
 }
 
+func (a blobPropertiesResponseAdapter) LastModified() time.Time {
+	return common.IffNotNil(a.GetPropertiesResponse.LastModified, time.Time{})
+}
+
 func (a blobPropertiesResponseAdapter) CacheControl() string {
 	return common.IffNotNil(a.GetPropertiesResponse.CacheControl, "")
 }
@@ -115,6 +127,10 @@ func (a blobPropertiesResponseAdapter) ContentMD5() []byte {
 	return a.GetPropertiesResponse.ContentMD5
 }
 
+func (a blobPropertiesResponseAdapter) ContentLength() int64 {
+	return common.IffNotNil(a.GetPropertiesResponse.ContentLength, 0)
+}
+
 func (a blobPropertiesResponseAdapter) BlobType() blob.BlobType {
 	return common.IffNotNil(a.GetPropertiesResponse.BlobType, "")
 }
@@ -172,6 +188,10 @@ func (a blobPropertiesAdapter) ContentMD5() []byte {
 	return a.BlobProperties.ContentMD5
 }
 
+func (a blobPropertiesAdapter) ContentLength() int64 {
+	return common.IffNotNil(a.BlobProperties.ContentLength, 0)
+}
+
 func (a blobPropertiesAdapter) BlobType() blob.BlobType {
 	return common.IffNotNil(a.BlobProperties.BlobType, "")
 }
@@ -199,6 +219,10 @@ func (a blobPropertiesAdapter) ArchiveStatus() blob.ArchiveStatus {
 	return common.IffNotNil(a.BlobProperties.ArchiveStatus, "")
 }
 
+func (a blobPropertiesAdapter) LastModified() time.Time {
+	return common.IffNotNil(a.BlobProperties.LastModified, time.Time{})
+}
+
 type shareFilePropertiesAdapter struct {
 	*sharefile.GetPropertiesResponse
 }
@@ -285,4 +309,4 @@ func (a shareDirectoryPropertiesAdapter) ContentMD5() []byte {
 
 func (a shareDirectoryPropertiesAdapter) ContentLength() int64 {
 	return 0
-}
+}

cmd/zc_traverser_blob.go

@@ -223,16 +223,17 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro
 			azcopyScanningLogger.Log(common.LogDebug, fmt.Sprintf("Root entity type: %s", getEntityType(blobProperties.Metadata)))
 		}
 
+		blobPropsAdapter := blobPropertiesResponseAdapter{blobProperties}
 		storedObject := newStoredObject(
 			preprocessor,
 			getObjectNameOnly(strings.TrimSuffix(blobURLParts.BlobName, common.AZCOPY_PATH_SEPARATOR_STRING)),
 			"",
-			getEntityType(blobProperties.Metadata),
-			*blobProperties.LastModified,
-			*blobProperties.ContentLength,
-			blobPropertiesResponseAdapter{blobProperties},
-			blobPropertiesResponseAdapter{blobProperties},
-			blobProperties.Metadata,
+			getEntityType(blobPropsAdapter.Metadata),
+			blobPropsAdapter.LastModified(),
+			blobPropsAdapter.ContentLength(),
+			blobPropsAdapter,
+			blobPropsAdapter,
+			blobPropsAdapter.Metadata,
 			blobURLParts.ContainerName,
 		)
 
@@ -340,6 +341,7 @@ func (t *blobTraverser) parallelList(containerClient *container.Client, containe
 						// try to get properties on the directory itself, since it's not listed in BlobItems
 						blobClient := containerClient.NewBlobClient(strings.TrimSuffix(*virtualDir.Name, common.AZCOPY_PATH_SEPARATOR_STRING))
 						pResp, err := blobClient.GetProperties(t.ctx, nil)
+						pbPropAdapter := blobPropertiesResponseAdapter{&pResp}
 						folderRelativePath := strings.TrimSuffix(*virtualDir.Name, common.AZCOPY_PATH_SEPARATOR_STRING)
 						folderRelativePath = strings.TrimPrefix(folderRelativePath, searchPrefix)
 						if err == nil {
@@ -348,11 +350,11 @@ func (t *blobTraverser) parallelList(containerClient *container.Client, containe
 								getObjectNameOnly(strings.TrimSuffix(*virtualDir.Name, common.AZCOPY_PATH_SEPARATOR_STRING)),
 								folderRelativePath,
 								common.EEntityType.Folder(),
-								*pResp.LastModified,
-								*pResp.ContentLength,
-								blobPropertiesResponseAdapter{&pResp},
-								blobPropertiesResponseAdapter{&pResp},
-								pResp.Metadata,
+								pbPropAdapter.LastModified(),
+								pbPropAdapter.ContentLength(),
+								pbPropAdapter,
+								pbPropAdapter,
+								pbPropAdapter.Metadata,
 								containerName,
 							)
 
@@ -466,8 +468,8 @@ func (t *blobTraverser) createStoredObjectForBlob(preprocessor objectMorpher, bl
 		getObjectNameOnly(*blobInfo.Name),
 		relativePath,
 		getEntityType(blobInfo.Metadata),
-		*blobInfo.Properties.LastModified,
-		*blobInfo.Properties.ContentLength,
+		adapter.LastModified(),
+		*adapter.BlobProperties.ContentLength,
 		adapter,
 		adapter, // adapter satisfies both interfaces
 		blobInfo.Metadata,

cmd/zc_traverser_blob_versions.go

@@ -87,17 +87,19 @@ func (t *blobVersionsTraverser) Traverse(preprocessor objectMorpher, processor o
 		if blobProperties == nil {
 			panic("isBlob should never be set if getting properties is an error")
 		}
+
+		blobPropsAdapter := blobPropertiesResponseAdapter{blobProperties}
 		blobURLParts.VersionID = versionID
 		storedObject := newStoredObject(
 			preprocessor,
 			getObjectNameOnly(strings.TrimSuffix(blobURLParts.BlobName, common.AZCOPY_PATH_SEPARATOR_STRING)),
 			"",
 			common.EEntityType.File(),
-			*blobProperties.LastModified,
-			*blobProperties.ContentLength,
-			blobPropertiesResponseAdapter{blobProperties},
-			blobPropertiesResponseAdapter{blobProperties},
-			blobProperties.Metadata,
+			blobPropsAdapter.LastModified(),
+			blobPropsAdapter.ContentLength(),
+			blobPropsAdapter,
+			blobPropsAdapter,
+			blobPropsAdapter.Metadata,
 			blobURLParts.ContainerName,
 		)
 		storedObject.blobVersionID = versionID

cmd/zt_traverser_blob_test.go

@@ -22,13 +22,18 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	blobservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	datalakedirectory "github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/directory"
 	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/Azure/azure-storage-azcopy/v10/mock_server"
 	"github.com/Azure/azure-storage-azcopy/v10/ste"
 	"github.com/stretchr/testify/assert"
 	"testing"
+	"time"
 )
 
 func TestIsSourceDirWithStub(t *testing.T) {
@@ -289,3 +294,80 @@ func TestGetEntityType(t *testing.T) {
 	a.Equal(common.EEntityType.Symlink(), entityType)
 
 }
+
+func TestManagedDiskProperties(t *testing.T) {
+	a := assert.New(t)
+
+	// Setup
+	// Mock the server
+	srv, close := mock_server.NewServer(mock_server.WithTransformAllRequestsToTestServerUrl())
+	defer close()
+
+	pbProp := &blob.GetPropertiesResponse{ContentLength: nil, LastModified: nil}
+	srv.AppendResponse(mock_server.WithStatusCode(200), mock_server.WithBody([]byte(getPageBlobProperties(pbProp))))
+
+	// Create a client
+	// Note: the key below is not a secret, this is the publicly documented Azurite key
+	accountName := "myfakeaccount"
+	accountKey := "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+	rawURL := fmt.Sprintf("https://%s.blob.core.windows.net/", accountName)
+
+	credential, err := blob.NewSharedKeyCredential(accountName, accountKey)
+	a.NoError(err)
+
+	client, err := blobservice.NewClientWithSharedKeyCredential(rawURL, credential,
+		&blobservice.ClientOptions{
+			ClientOptions: azcore.ClientOptions{
+				Transport: srv,
+			}})
+	a.NoError(err)
+
+	containerName := generateContainerName()
+	containerClient := client.NewContainerClient(containerName)
+
+	blobName := generateBlobName()
+	blobClient := containerClient.NewPageBlobClient(blobName)
+
+	prop, err := blobClient.GetProperties(ctx, nil)
+	a.NoError(err)
+	a.Nil(prop.LastModified)
+	a.NotNil(prop.ContentLength) // note:content length will never be nil as the service calculates the size of the blob and stores it in this header
+
+	propAdapter := blobPropertiesResponseAdapter{GetPropertiesResponse: &prop}
+	a.Equal(propAdapter.LastModified(), time.Time{})
+	a.NotNil(prop.ContentLength) // see note from above
+}
+
+func getPageBlobProperties(properties *blob.GetPropertiesResponse) string {
+	// these properties have been pulled from https://learn.microsoft.com/en-us/rest/api/storageservices/get-blob-properties
+	// with modification to date, content length and last modified time
+	body := "x-ms-blob-type: PageBlob" +
+		"x-ms-lease-status: unlocked" +
+		"x-ms-lease-state: available" +
+		getContentLength(properties) +
+		"Content-Type: text/plain; charset=UTF-8" +
+		fmt.Sprintf("Date: %s", time.Now().String()) +
+		"ETag: \"0x8CAE97120C1FF22\"" +
+		"Accept-Ranges: bytes" +
+		"x-ms-blob-committed–block-count: 1" +
+		"x-ms-version: 2015-02-21" +
+		getLMT(properties) +
+		"Server: Windows-Azure-Blob/1.0 Microsoft-HTTPAPI/2.0"
+	return body
+}
+
+func getLMT(response *blob.GetPropertiesResponse) string {
+	if response.LastModified == nil {
+		return ""
+	} else {
+		return fmt.Sprintf("Last-Modified: %s", response.LastModified.String())
+	}
+}
+
+func getContentLength(response *blob.GetPropertiesResponse) string {
+	if response.ContentLength == nil {
+		return ""
+	} else {
+		return fmt.Sprintf("Content-Length: %d", response.ContentLength)
+	}
+}

e2etest/config.go

@@ -123,13 +123,16 @@ func (AccountType) GCP() AccountType                          { return AccountTy
 func (AccountType) Azurite() AccountType                      { return AccountType(8) }
 func (AccountType) ManagedDiskSnapshot() AccountType          { return AccountType(9) }
 func (AccountType) ManagedDiskSnapshotOAuth() AccountType     { return AccountType(10) }
+func (AccountType) LargeManagedDiskSnapshot() AccountType     { return AccountType(11) }
+func (AccountType) LargeManagedDisk() AccountType             { return AccountType(12) }
 
 func (o AccountType) String() string {
 	return enum.StringInt(o, reflect.TypeOf(o))
 }
 
 func (o AccountType) IsManagedDisk() bool {
-	return o == o.StdManagedDisk() || o == o.OAuthManagedDisk() || o == o.ManagedDiskSnapshot() || o == o.ManagedDiskSnapshotOAuth()
+	return o == o.StdManagedDisk() || o == o.OAuthManagedDisk() || o == o.ManagedDiskSnapshot() || o == o.ManagedDiskSnapshotOAuth() ||
+		o == o.LargeManagedDiskSnapshot() || o == o.LargeManagedDisk()
 }
 
 func (o AccountType) IsBlobOnly() bool {
@@ -144,7 +147,7 @@ type ManagedDiskConfig struct {
 	ResourceGroupName string
 	DiskName          string
 
-	oauth             AccessToken
+	oauth      AccessToken
 	isSnapshot bool
 }
 
@@ -159,12 +162,17 @@ func (gim GlobalInputManager) GetMDConfig(accountType AccountType) (*ManagedDisk
 		mdConfigVar = "AZCOPY_E2E_STD_MANAGED_DISK_CONFIG"
 	case EAccountType.OAuthManagedDisk():
 		mdConfigVar = "AZCOPY_E2E_OAUTH_MANAGED_DISK_CONFIG"
+	case EAccountType.LargeManagedDisk():
+		mdConfigVar = "AZCOPY_E2E_LARGE_MANAGED_DISK_CONFIG"
 	case EAccountType.ManagedDiskSnapshot():
 		mdConfigVar = "AZCOPY_E2E_STD_MANAGED_DISK_SNAPSHOT_CONFIG"
 		isSnapshot = true
 	case EAccountType.ManagedDiskSnapshotOAuth():
 		mdConfigVar = "AZCOPY_E2E_OAUTH_MANAGED_DISK_SNAPSHOT_CONFIG"
 		isSnapshot = true
+	case EAccountType.LargeManagedDiskSnapshot():
+		mdConfigVar = "AZCOPY_E2E_LARGE_MANAGED_DISK_SNAPSHOT_CONFIG"
+		isSnapshot = true
 	default:
 		return nil, fmt.Errorf("account type %s is invalid for GetMDConfig", accountType.String())
 	}

e2etest/declarativeScenario.go

@@ -113,8 +113,10 @@ func (s *scenario) Run() {
 	if s.destAccountType.IsManagedDisk() {
 		s.a.Assert(s.destAccountType, notEquals(), EAccountType.StdManagedDisk(), "Upload is not supported in MD testing yet")
 		s.a.Assert(s.destAccountType, notEquals(), EAccountType.OAuthManagedDisk(), "Upload is not supported in MD testing yet")
+		s.a.Assert(s.destAccountType, notEquals(), EAccountType.LargeManagedDisk(), "Upload is not supported in MD testing yet")
 		s.a.Assert(s.destAccountType, notEquals(), EAccountType.ManagedDiskSnapshot(), "Cannot upload to a MD snapshot")
 		s.a.Assert(s.destAccountType, notEquals(), EAccountType.ManagedDiskSnapshotOAuth(), "Cannot upload to a MD snapshot")
+		s.a.Assert(s.destAccountType, notEquals(), EAccountType.LargeManagedDiskSnapshot(), "Cannot upload to a MD snapshot")
 		s.a.Assert(true, equals(), s.fromTo.From() == common.ELocation.Blob() || s.fromTo.From() == common.ELocation.BlobFS())
 	}
 

e2etest/helpers.go

@@ -27,6 +27,7 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/binary"
+	"flag"
 	"fmt"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
@@ -78,6 +79,8 @@ const (
 	defaultBlobFSFileSizeInBytes = 1000
 )
 
+var runLocallyOnly = flag.Bool("local-tests", false, "Tests with this flag are run locally only")
+
 func pointerTo[T any](in T) *T {
 	return &in
 }