Fix SAS times + Upload logs for other types of failed operations

e2etest/newe2e_sas_generation.go

@@ -77,8 +77,9 @@ func (vals GenericServiceSignatureValues) withDefaults() GenericServiceSignature
 	out := vals
 
 	SetIfZero(&out.Protocol, blobsas.ProtocolHTTPS)
-	SetIfZero(&out.StartTime, time.Now().Add(-time.Minute*30))
-	SetIfZero(&out.ExpiryTime, out.StartTime.Add(time.Minute*30))
+	//time.Now().Add()
+	SetIfZero(&out.StartTime, time.Now().UTC().Add(-time.Minute*30))
+	SetIfZero(&out.ExpiryTime, time.Now().UTC().Add(-time.Minute*30))
 	SetIfZero(&out.Permissions, (&blobsas.ContainerPermissions{
 		Read: true, Add: true, Create: true, Write: true, Delete: true, List: true,
 	}).String())
@@ -181,8 +182,8 @@ func (vals GenericAccountSignatureValues) withDefaults() GenericAccountSignature
 	out := vals
 
 	SetIfZero(&out.Protocol, blobsas.ProtocolHTTPS)
-	SetIfZero(&out.StartTime, time.Now())
-	SetIfZero(&out.ExpiryTime, out.StartTime.Add(time.Hour*24))
+	SetIfZero(&out.StartTime, time.Now().UTC().Add(-time.Minute*30))
+	SetIfZero(&out.ExpiryTime, time.Now().UTC().Add(time.Minute*30))
 	SetIfZero(&out.Permissions, (&blobsas.AccountPermissions{
 		Read: true, Add: true, Create: true, Write: true, Delete: true, List: true,
 	}).String())

e2etest/newe2e_task_resourcemanagement.go

@@ -116,7 +116,7 @@ func ValidateResource[T ResourceManager](a Asserter, target T, definition Matche
 
 			exists := cRes.Exists()
 			if definition.ShouldExist() != exists {
-				a.Assert(cRes.ContainerName()+": object must "+common.Iff(definition.ShouldExist(), "not exist", "exist"), Equal{}, exists, false)
+				a.Assert(cRes.ContainerName()+": object must "+common.Iff(definition.ShouldExist(), "exist", "not exist"), Equal{}, exists, false)
 				return
 			}
 
@@ -147,7 +147,7 @@ func ValidateResource[T ResourceManager](a Asserter, target T, definition Matche
 
 			exists := objMan.Exists()
 			if objDef.ShouldExist() != exists {
-				a.Assert(objMan.ObjectName()+": object must "+common.Iff(objDef.ShouldExist(), "not exist", "exist"), Equal{}, objDef.ShouldExist(), exists)
+				a.Assert(objMan.ObjectName()+": object must "+common.Iff(objDef.ShouldExist(), "exist", "not exist"), Equal{}, objDef.ShouldExist(), exists)
 				return
 			}
 
@@ -214,6 +214,7 @@ func ValidateListOutput(a Asserter, stdout AzCopyStdout, expectedObjects map[AzC
 	if dryrunner, ok := a.(DryrunAsserter); ok && dryrunner.Dryrun() {
 		return
 	}
+	a.HelperMarker().Helper()
 
 	listStdout, ok := stdout.(*AzCopyParsedListStdout)
 	a.AssertNow("stdout must be AzCopyParsedListStdout", Equal{}, ok, true)

e2etest/newe2e_task_runazcopy.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/google/uuid"
 	"io"
 	"io/fs"
 	"os"
@@ -206,6 +207,7 @@ func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *Az
 	if a.Dryrun() {
 		return nil, &AzCopyJobPlan{}
 	}
+	a.HelperMarker().Helper()
 	var flagMap map[string]string
 	var envMap map[string]string
 
@@ -291,34 +293,39 @@ func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *Az
 		common.Iff[Assertion](commandSpec.ShouldFail, Not{Equal{}}, Equal{}),
 		0, command.ProcessState.ExitCode())
 
+	if err != nil {
+		a.Log(out.String())
+	}
+
 	a.Cleanup(func(a ScenarioAsserter) {
-		if stdout, ok := out.(*AzCopyParsedCopySyncRemoveStdout); ok {
-			UploadLogs(a, stdout, stderr, DerefOrZero(commandSpec.Environment.LogLocation))
-			_ = os.RemoveAll(DerefOrZero(commandSpec.Environment.LogLocation))
-		}
+		UploadLogs(a, out, stderr, DerefOrZero(commandSpec.Environment.LogLocation))
+		_ = os.RemoveAll(DerefOrZero(commandSpec.Environment.LogLocation))
 	})
 
 	return out, &AzCopyJobPlan{}
 }
 
-func UploadLogs(a ScenarioAsserter, stdout *AzCopyParsedCopySyncRemoveStdout, stderr *bytes.Buffer, logDir string) {
+func UploadLogs(a ScenarioAsserter, stdout AzCopyStdout, stderr *bytes.Buffer, logDir string) {
 	defer func() {
 		if err := recover(); err != nil {
 			fmt.Println("Log cleanup failed", err, "\n", string(debug.Stack()))
 		}
 	}()
 
-	logPath := GlobalConfig.AzCopyExecutableConfig.LogDropPath
-	if logPath == "" || !a.Failed() {
+	logDropPath := GlobalConfig.AzCopyExecutableConfig.LogDropPath
+	if logDropPath == "" || !a.Failed() {
 		return
 	}
 
 	// sometimes, the log dir cannot be copied because the destination is on another drive. So, we'll copy the files instead by hand.
 	files, err := os.ReadDir(logDir)
 	a.NoError("Failed to read log dir", err)
 	jobId := ""
-	if stdout.InitMsg.JobID != "" {
-		jobId = stdout.InitMsg.JobID
+
+	if jobStdout, ok := stdout.(*AzCopyParsedCopySyncRemoveStdout); ok {
+		if jobStdout.InitMsg.JobID != "" {
+			jobId = jobStdout.InitMsg.JobID
+		}
 	} else {
 		for _, file := range files { // first, find the job ID
 			if strings.HasSuffix(file.Name(), ".log") {
@@ -328,8 +335,13 @@ func UploadLogs(a ScenarioAsserter, stdout *AzCopyParsedCopySyncRemoveStdout, st
 		}
 	}
 
+	if jobId == "" {
+		// If we still don't have a job ID, let's make one up. Maybe the job never started, or this isn't a copy/sync/remove job anyway.
+		jobId = uuid.NewString()
+	}
+
 	// Create the destination log directory
-	destLogDir := filepath.Join(logPath, jobId)
+	destLogDir := filepath.Join(logDropPath, jobId)
 	err = os.MkdirAll(destLogDir, os.ModePerm|os.ModeDir)
 	a.NoError("Failed to create log dir", err)
 

e2etest/zt_newe2e_list_test.go

@@ -18,7 +18,6 @@ func (s *ListSuite) Scenario_ListBasic(svm *ScenarioVariationManager) {
 	acct := GetAccount(svm, PrimaryStandardAcct)
 	srcService := acct.GetService(svm, common.ELocation.Blob())
 
-	svm.InsertVariationSeparator(":")
 	body := NewRandomObjectContentContainer(svm, SizeFromString("1K"))
 	// Scale up from service to object
 	srcObj := CreateResource[ObjectResourceManager](svm, srcService, ResourceDefinitionObject{
@@ -55,7 +54,6 @@ func (s *ListSuite) Scenario_ListRunningTally(svm *ScenarioVariationManager) {
 	acct := GetAccount(svm, PrimaryStandardAcct)
 	srcService := acct.GetService(svm, common.ELocation.Blob())
 
-	svm.InsertVariationSeparator(":")
 	body := NewRandomObjectContentContainer(svm, SizeFromString("1K"))
 	// Scale up from service to object
 	srcObj := CreateResource[ObjectResourceManager](svm, srcService, ResourceDefinitionObject{
@@ -93,8 +91,6 @@ func (s *ListSuite) Scenario_ListRunningTally(svm *ScenarioVariationManager) {
 func (s *ListSuite) Scenario_ListVersionIdNoAdditionalVersions(svm *ScenarioVariationManager) {
 	acct := GetAccount(svm, PrimaryStandardAcct)
 	srcService := acct.GetService(svm, common.ELocation.Blob())
-
-	svm.InsertVariationSeparator(":")
 	srcContainer := CreateResource[ContainerResourceManager](svm, srcService, ResourceDefinitionContainer{})
 
 	// Create expected objects