perf-test.yaml

trigger: none
pr: none

stages:
- stage: Smallfiles
  jobs:
    - job: PerformanceTest
      timeoutInMinutes: 720
      strategy:
        matrix:
          Ubuntu-22:
            imageName: "azcopyPerfTestUbuntu22.04"
            Description: "AzCopy Perf Test"

      pool:
        name: "AzCopyPerfTestUbuntu"
        demands:
          - ImageOverride -equals $(imageName)

      variables:
      - group: AzCopyPerfTestTargets
      - name: localPath
        value: "/mnt/storage"

      steps:
        - script: |
            echo $(Description)
            hostnamectl
          displayName: 'Print Agent Info'
        
        - task: GoTool@0
          inputs:
            version: '1.22.3'

        - script: |
            go build -o $GOROOT/bin/azcopy
            azcopy --version
          displayName: 'Build Azcopy'

        - script: |
            time azcopy copy $(Blob2BlobSmallFilesSrc) $(Blob2BlobSmallFilesDst) --recursive --check-length=false --log-level=ERROR
          displayName: 'Blob2Blob - Small Files'
          condition: always()
          env:
            AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
            AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
            AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
            AZCOPY_CONCURRENCY_VALUE: "256"
            AZCOPY_SHOW_PERF_STATES: "1"

        - script: |
            time azcopy copy $(Blob2BlobSmallAndMedFilesSrc) $(Blob2BlobSmallAndMedFilesDst) --recursive --block-size-mb=128 --log-level=ERROR
          displayName: 'Blob2Blob - Small to Medium sized files'
          condition: always()
          env:
            AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
            AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
            AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
            AZCOPY_CONCURRENCY_VALUE: "256"
            AZCOPY_SHOW_PERF_STATES: "1"

        - script: |
            time azcopy copy $(Blob2BlobLargeFilesSrc) $(Blob2BlobLargeFilesDst) --recursive --block-size-mb=128 --log-level=ERROR --cap-mbps=40000
          displayName: 'Blob2Blob - Large Files'
          condition: always()
          env:
            AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
            AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
            AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs
            AZCOPY_CONCURRENCY_VALUE: "256"
            AZCOPY_SHOW_PERF_STATES: "1"

        - script: |
            sudo mkdir -m 777 $(localPath)/smallFiles/
            time azcopy copy $(Blob2BlobSmallFilesSrc) /dev/null --recursive --check-length=false --log-level=ERROR
          displayName: 'Download - Small Files'
          condition: always()
          env:
            AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
            AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
            AZCOPY_SHOW_PERF_STATES: "1"
            AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs

        - script: |
            time azcopy bench $(Blob2BlobSmallFilesDst) --size-per-file=5k --file-count=8000000 --check-length=false --log-level=ERROR --delete-test-data=false
            sudo rm -rf $(localPath)/*
          displayName: 'Upload - Small Files'
          condition: always()
          env:
            AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
            AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
            AZCOPY_SHOW_PERF_STATES: "1"
            AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs

        - script: |
            sudo mkdir -m 777 $(localPath)/smallToMediumFiles/
            time azcopy copy $(Blob2BlobSmallAndMedFilesSrc) $(localPath)/smallToMediumFiles --recursive --log-level=ERROR
          displayName: 'Download - Small to Medium sized files'
          condition: always()
          env:
            AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
            AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
            AZCOPY_SHOW_PERF_STATES: "1"
            AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs

        - script: |
            time azcopy copy $(localPath)/smallToMediumFiles/ $(Blob2BlobSmallAndMedFilesDst) --recursive --log-level=ERROR
            sudo rm -rf $(localPath)/*
          displayName: 'Upload - Small to Medium sized files'
          condition: always()
          env:
            AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
            AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
            AZCOPY_SHOW_PERF_STATES: "1"
            AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs

        - script: |
            sudo mkdir -m 777 $(localPath)/largeFiles/
            time azcopy copy $(Blob2BlobLargeFilesSrc) /dev/null --recursive --log-level=ERROR
          displayName: 'Download - Large files'
          condition: always()
          env:
            AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
            AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
            AZCOPY_SHOW_PERF_STATES: "1"
            AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs

        - script: |
            time azcopy bench $(Blob2BlobLargeFilesDst) --log-level=ERROR --size-per-file=50G --file-count=50 --put-md5=false --delete-test-data=false
            sudo rm -rf $(localPath)/*
          displayName: 'Upload - Large files'
          condition: always()
          env:
            AZCOPY_AUTO_LOGIN_TYPE: $(AZCOPY_AUTO_LOGIN_TYPE)
            AZCOPY_MSI_CLIENT_ID: $(AZCOPY_MSI_CLIENT_ID)
            AZCOPY_SHOW_PERF_STATES: "1"
            AZCOPY_LOG_LOCATION: $(Build.ArtifactStagingDirectory)/logs

        - task: PublishBuildArtifacts@1
          condition: always()
          inputs:
            pathToPublish: $(Build.ArtifactStagingDirectory)
            artifactName: Logs

        - script: |
            curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
            az login --identity --username $(AZCOPY_MSI_CLIENT_ID)
            for container in `az storage container list --account-name $(DestinationAccount) --query "[*].[name]" --output tsv --auth-mode login`; do
              az storage container delete --account-name $(DestinationAccount) --name $container --auth-mode login 
            done
          displayName: 'Clean destination storage Account'
          condition: always()