contoso-traders-cloud-testing.yml

name: contoso-traders-cloud-testing

on:
  workflow_dispatch:
  push:
    branches: ["main"]
    paths-ignore: ["docs/**", "demo-scripts/**"]

env:
  ACR_NAME: contosotradersacr
  AKS_CLUSTER_NAME: contoso-traders-aks
  AKS_CPU_LIMIT: 250m
  AKS_DNS_LABEL: contoso-traders-products
  AKS_MEMORY_LIMIT: 256Mi
  AKS_NODES_RESOURCE_GROUP_NAME: contoso-traders-aks-nodes-rg
  AKS_REPLICAS: "1"
  AKS_SECRET_NAME_ACR_PASSWORD: contoso-traders-acr-password
  AKS_SECRET_NAME_KV_ENDPOINT: contoso-traders-kv-endpoint
  AKS_SECRET_NAME_MI_CLIENTID: contoso-traders-mi-clientid
  AZURE_AD_APP_NAME: contoso-traders-cloud-testing-app
  CARTS_ACA_NAME: contoso-traders-carts
  CARTS_ACR_REPOSITORY_NAME: contosotradersapicarts
  CARTS_INTERNAL_ACA_NAME: contoso-traders-intcarts
  CDN_PROFILE_NAME: contoso-traders-cdn
  CHAOS_AKS_EXPERIMENT_NAME: contoso-traders-chaos-aks-experiment
  KV_NAME: contosotraderskv
  LOAD_TEST_SERVICE_NAME: contoso-traders-loadtest
  MSGRAPH_API_ID: 00000003-0000-0000-c000-000000000000
  MSGRAPH_API_PERMISSION_EMAIL: 64a6cdd6-aab1-4aaf-94b8-3cc8405e90d0=Scope
  MSGRAPH_API_PERMISSION_USER_READ: e1fe6dd8-ba31-4d61-89e7-88639da4683d=Scope
  PRODUCTS_ACR_REPOSITORY_NAME: contosotradersapiproducts
  PRODUCTS_DB_NAME: productsdb
  PRODUCTS_DB_SERVER_NAME: contoso-traders-products
  PRODUCTS_DB_USER_NAME: localadmin
  PRODUCT_DETAILS_CONTAINER_NAME: product-details
  PRODUCT_IMAGES_STORAGE_ACCOUNT_NAME: contosotradersimg
  PRODUCT_LIST_CONTAINER_NAME: product-list
  PRODUCTS_CDN_ENDPOINT_NAME: contoso-traders-images
  RESOURCE_GROUP_NAME: contoso-traders-rg
  STORAGE_ACCOUNT_NAME: contosotradersimg
  UI_CDN_ENDPOINT_NAME: contoso-traders-ui2
  UI_STORAGE_ACCOUNT_NAME: contosotradersui2
  USER_ASSIGNED_MANAGED_IDENTITY_NAME: contoso-traders-mi-kv-access

jobs:
  provision:
    runs-on: ubuntu-22.04
    env:
      AADUSERNAME: ${{ secrets.AADUSERNAME }}
      AADPASSWORD: ${{ secrets.AADPASSWORD }}
    outputs:
      azureAdAppClientId: ${{ steps.get-azureAdAppClientId.outputs.azureAdAppClientId }}
      azureAdAppObjId: ${{ steps.get-azureAdAppObjId.outputs.azureAdAppObjId }}
      cartsApiEndpoint: ${{ steps.get-cartsApiEndpoint.outputs.cartsApiEndpoint }}
      productsApiEndpoint: ${{ steps.get-productsApiEndpoint.outputs.productsApiEndpoint }}
      uiCdnEndpoint: ${{ steps.get-uiCdnEndpoint.outputs.uiCdnEndpoint }}
    concurrency:
      group: provision
      cancel-in-progress: true
    steps:
      - name: checkout code
        uses: actions/checkout@v4
      - name: azure login
        uses: azure/login@v1
        with:
          creds: ${{ secrets.SERVICEPRINCIPAL }}
      # section #0: optional configuration of the Azure AD app.
      # create the Azure AD application (and update it if it already exists).
      # note: This is an idempotent operation.
      - name: create/update azure active directory app
        uses: azure/CLI@v1
        if: ${{ env.AADUSERNAME != '' && env.AADPASSWORD != '' }}
        with:
          inlineScript: az ad app create --display-name ${{ env.AZURE_AD_APP_NAME}}${{ vars.SUFFIX }} --sign-in-audience AzureADandPersonalMicrosoftAccount
      - name: get azure ad app's object id
        uses: azure/CLI@v1
        if: ${{ env.AADUSERNAME != '' && env.AADPASSWORD != '' }}
        id: get-azureAdAppObjId
        with:
          inlineScript: echo "azureAdAppObjId"="$(az ad app list --display-name ${{ env.AZURE_AD_APP_NAME }}${{ vars.SUFFIX }} --query [].id -o tsv)" >> $GITHUB_OUTPUT
      - name: get azure ad app's client id
        uses: azure/CLI@v1
        if: ${{ env.AADUSERNAME != '' && env.AADPASSWORD != '' }}
        id: get-azureAdAppClientId
        with:
          inlineScript: echo "azureAdAppClientId"="$(az ad app list --display-name ${{ env.AZURE_AD_APP_NAME }}${{ vars.SUFFIX }} --query [].appId -o tsv)" >> $GITHUB_OUTPUT
      - name: register app as a spa
        uses: azure/CLI@v1
        if: ${{ env.AADUSERNAME != '' && env.AADPASSWORD != '' }}
        with:
          inlineScript: |
            az rest \
              --method PATCH \
              --uri https://graph.microsoft.com/v1.0/applications/${{ steps.get-azureAdAppObjId.outputs.azureAdAppObjId }} \
              --headers 'Content-Type=application/json' \
              --body '{"spa":{"redirectUris":["https://localhost:3000/authcallback","http://localhost:3000/authcallback","https://production.contosotraders.com/authcallback","https://cloudtesting.contosotraders.com/authcallback"]}}'
      - name: enable issuance of id, access tokens
        uses: azure/CLI@v1
        if: ${{ env.AADUSERNAME != '' && env.AADPASSWORD != '' }}
        with:
          inlineScript: az ad app update --id ${{ steps.get-azureAdAppObjId.outputs.azureAdAppObjId }} --enable-access-token-issuance true --enable-id-token-issuance true
      - name: enable email claim in access token
        uses: azure/CLI@v1
        if: ${{ env.AADUSERNAME != '' && env.AADPASSWORD != '' }}
        with:
          inlineScript: az ad app update --id ${{ steps.get-azureAdAppObjId.outputs.azureAdAppObjId }} --optional-claims "{\"accessToken\":[{\"name\":\"email\",\"essential\":false}]}"
      # note: requesting MS Graph permissions in Azure AD app unfortunately isn't idempotent.
      # Even, if you have already requested the permissions, it'll keep adding to the list of requested permissions until you hit limit on max permissions requested.
      # Details: https://github.com/Azure/azure-cli/issues/24512
      - name: delete any requested Microsoft Graph permissions
        uses: azure/CLI@v1
        if: ${{ env.AADUSERNAME != '' && env.AADPASSWORD != '' }}
        with:
          inlineScript: |
            az ad app permission delete \
              --id ${{ steps.get-azureAdAppObjId.outputs.azureAdAppObjId }} \
              --api ${{ env.MSGRAPH_API_ID }}
      - name: request Microsoft Graph permissions
        uses: azure/CLI@v1
        if: ${{ env.AADUSERNAME != '' && env.AADPASSWORD != '' }}
        with:
          inlineScript: |
            az ad app permission add \
              --id ${{ steps.get-azureAdAppObjId.outputs.azureAdAppObjId }} \
              --api ${{ env.MSGRAPH_API_ID }} \
              --api-permissions ${{ env.MSGRAPH_API_PERMISSION_USER_READ }} ${{ env.MSGRAPH_API_PERMISSION_EMAIL }}
      #
      # section #1: provisioning the resources on Azure using bicep templates
      #
      # The first step is to create the resource group: `contoso-traders-rg`.
      # The below step can also be manually executed as follows:
      # az deployment sub create --location {LOCATION} --template-file .\createResourceGroup.bicep
      # Note: You can specify any location for `{LOCATION}`. It's the region where the deployment metadata will be stored, and not
      # where the resource groups will be deployed.
      - name: create resource group
        uses: Azure/arm-deploy@v1
        with:
          scope: subscription
          region: ${{ vars.DEPLOYMENTREGION }}
          template: ./iac/createResourceGroup.bicep
          parameters: rgName=${{ env.RESOURCE_GROUP_NAME }} suffix=${{ vars.SUFFIX }} rgLocation=${{ vars.DEPLOYMENTREGION }}
      # Next step is to deploy the Azure resources to the resource group `contoso-traders-rg` created above. The deployed resources
      # include storage accounts, function apps, app services cosmos db, and service bus etc.
      # The below step can also be manually executed as follows:
      # az deployment group create -g contoso-traders-rg --template-file .\createResources.bicep --parameters .\createResources.parameters.json
      # Note: The `createResources.parameters.json` file contains the parameters for the deployment; specifically the environment name.
      # You can modify the parameters to customize the deployment.
      # Note: The bicep template outputs are not shown in the logs. You can extract the outputs as shown here:
      # https://github.com/Azure/arm-deploy#another-example-on-how-to-use-this-action-to-get-the-output-of-arm-template
      - name: create resources
        uses: Azure/arm-deploy@v1
        with:
          scope: resourcegroup
          region: ${{ vars.DEPLOYMENTREGION }}
          resourceGroupName: ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }}
          template: ./iac/createResources.bicep
          parameters: ./iac/createResources.parameters.json suffix=${{ vars.SUFFIX }} sqlPassword=${{ secrets.SQLPASSWORD }}
      # Add the logged-in service principal to the key vault access policy
      - name: add service principal to kv access policy
        uses: azure/CLI@v1
        with:
          inlineScript: az keyvault set-policy -n ${{ env.KV_NAME }}${{ vars.SUFFIX }} -g ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} --secret-permissions get list set --object-id $(az ad sp show --id $(az account show --query "user.name" -o tsv) --query "id" -o tsv)
      # The AKS agent pool needs to be assigned the user-assigned managed identity created (which has kv access)
      - name: assign user-assigned managed-identity to aks agentpool
        uses: azure/CLI@v1
        with:
          inlineScript: |
            az vmss identity assign \
              --identities $(az identity show -g ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} --name ${{ env.USER_ASSIGNED_MANAGED_IDENTITY_NAME }}${{ vars.SUFFIX }} --query "id" -o tsv) \
              --ids $(az vmss list -g ${{ env.AKS_NODES_RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} --query "[0].id" -o tsv) \
      # Seed the DBs and storage accounts
      - name: seed products db
        uses: azure/sql-action@v2.2
        with:
          connection-string: Server=tcp:${{ env.PRODUCTS_DB_SERVER_NAME }}${{ vars.SUFFIX }}.database.windows.net,1433;Initial Catalog=${{ env.PRODUCTS_DB_NAME }};Persist Security Info=False;User ID=${{ env.PRODUCTS_DB_USER_NAME }};Password=${{ secrets.SQLPASSWORD }};MultipleActiveResultSets=False;Encrypt=True;TrustServerCertificate=False;Connection Timeout=30;
          path: ./src/ContosoTraders.Api.Products/Migration/productsdb.sql
      - name: seed product image (product details)
        uses: azure/CLI@v1
        with:
          inlineScript: az storage blob sync --account-name '${{ env.PRODUCT_IMAGES_STORAGE_ACCOUNT_NAME }}${{ vars.SUFFIX }}' -c '${{ env.PRODUCT_DETAILS_CONTAINER_NAME }}' -s 'src/ContosoTraders.Api.Images/product-details'
      - name: seed product image (product list)
        uses: azure/CLI@v1
        with:
          inlineScript: az storage blob sync --account-name '${{ env.PRODUCT_IMAGES_STORAGE_ACCOUNT_NAME }}${{ vars.SUFFIX }}' -c '${{ env.PRODUCT_LIST_CONTAINER_NAME }}' -s 'src/ContosoTraders.Api.Images/product-list'
      - name: purge product images cdn endpoint
        uses: azure/CLI@v1
        with:
          inlineScript: az cdn endpoint purge --no-wait --content-paths '/*' -n '${{ env.PRODUCTS_CDN_ENDPOINT_NAME }}${{ vars.SUFFIX }}' -g '${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }}' --profile-name '${{ env.CDN_PROFILE_NAME }}${{ vars.SUFFIX }}'
      - name: extract acr password
        uses: azure/CLI@v1
        id: extract-acr-password
        with:
          inlineScript: |
            acrPassword=$(az acr credential show -n ${{ env.ACR_NAME }}${{ vars.SUFFIX }} -g ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} --query "passwords[0].value" --output tsv)
            echo "::add-mask::$acrPassword"
            echo acrPassword=$acrPassword >> $GITHUB_OUTPUT
      - name: azure container registry login
        uses: azure/docker-login@v1
        with:
          login-server: ${{ env.ACR_NAME }}${{ vars.SUFFIX }}.azurecr.io
          username: ${{ env.ACR_NAME }}${{ vars.SUFFIX }}
          password: ${{ steps.extract-acr-password.outputs.acrPassword }}
      - name: set aks context
        uses: azure/aks-set-context@v3
        with:
          resource-group: ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }}
          cluster-name: ${{ env.AKS_CLUSTER_NAME }}${{ vars.SUFFIX }}
      #
      # section #2: deploy the carts api
      #
      - name: docker build
        run: docker build src -f ./src/ContosoTraders.Api.Carts/Dockerfile -t ${{ env.ACR_NAME }}${{ vars.SUFFIX }}.azurecr.io/${{ env.CARTS_ACR_REPOSITORY_NAME }}:latest -t ${{ env.ACR_NAME }}${{ vars.SUFFIX }}.azurecr.io/${{ env.CARTS_ACR_REPOSITORY_NAME }}:${{ github.sha }}
      - name: docker push (to acr)
        run: docker push --all-tags ${{ env.ACR_NAME }}${{ vars.SUFFIX }}.azurecr.io/${{ env.CARTS_ACR_REPOSITORY_NAME }}
      - name: deploy to aca
        uses: azure/CLI@v1
        with:
          inlineScript: |
            az config set extension.use_dynamic_install=yes_without_prompt
            az containerapp update -n ${{ env.CARTS_ACA_NAME }}${{ vars.SUFFIX }} -g ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} --image ${{ env.ACR_NAME }}${{ vars.SUFFIX }}.azurecr.io/${{ env.CARTS_ACR_REPOSITORY_NAME }}:${{ github.sha }}
      - name: deploy to aca (internal)
        uses: azure/CLI@v1
        with:
          inlineScript: |
            az config set extension.use_dynamic_install=yes_without_prompt
            az containerapp update -n ${{ env.CARTS_INTERNAL_ACA_NAME }}${{ vars.SUFFIX }} -g ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} --image ${{ env.ACR_NAME }}${{ vars.SUFFIX }}.azurecr.io/${{ env.CARTS_ACR_REPOSITORY_NAME }}:${{ github.sha }}
      - name: get carts api endpoint
        uses: azure/CLI@v1
        id: get-cartsApiEndpoint
        with:
          inlineScript: echo "cartsApiEndpoint"="$(az keyvault secret show --vault-name ${{ env.KV_NAME }}${{ vars.SUFFIX }} --name cartsApiEndpoint --query value -o tsv)" >> $GITHUB_OUTPUT

      #
      # section #3: deploy the products api
      #
      - name: install helm
        uses: Azure/setup-helm@v3
        id: install-helm
        with:
          version: v3.9.0
      - name: docker build
        run: docker build src -f ./src/ContosoTraders.Api.Products/Dockerfile -t ${{ env.ACR_NAME }}${{ vars.SUFFIX }}.azurecr.io/${{ env.PRODUCTS_ACR_REPOSITORY_NAME }}:latest -t ${{ env.ACR_NAME }}${{ vars.SUFFIX }}.azurecr.io/${{ env.PRODUCTS_ACR_REPOSITORY_NAME }}:${{ github.sha }}
      - name: docker push (to acr)
        run: docker push --all-tags ${{ env.ACR_NAME }}${{ vars.SUFFIX }}.azurecr.io/${{ env.PRODUCTS_ACR_REPOSITORY_NAME }}
      - name: setup kubectl
        uses: azure/setup-kubectl@v3
      - name: create kubernetes secret (acr password)
        uses: Azure/k8s-create-secret@v4
        with:
          secret-name: ${{ env.AKS_SECRET_NAME_ACR_PASSWORD }}
          container-registry-url: ${{ env.ACR_NAME }}${{ vars.SUFFIX }}.azurecr.io
          container-registry-username: ${{ env.ACR_NAME }}${{ vars.SUFFIX }}
          container-registry-password: ${{ steps.extract-acr-password.outputs.acrPassword }}
      - name: get managedIdentityClientId
        uses: azure/CLI@v1
        id: get-managedIdentityClientId
        with:
          inlineScript: echo "managedIdentityClientId"="$(az identity show -g ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} --name ${{ env.USER_ASSIGNED_MANAGED_IDENTITY_NAME }}${{ vars.SUFFIX }} --query "clientId" -o tsv)" >> $GITHUB_OUTPUT
      - name: create kubernetes secret (kv endpoint)
        uses: Azure/k8s-create-secret@v4
        with:
          secret-type: "generic"
          secret-name: ${{ env.AKS_SECRET_NAME_KV_ENDPOINT }}
          string-data: '{ "${{ env.AKS_SECRET_NAME_KV_ENDPOINT }}" : "https://${{ env.KV_NAME }}${{ vars.SUFFIX }}.vault.azure.net/" }'
      - name: create kubernetes secret (managed identity client id)
        uses: Azure/k8s-create-secret@v4
        with:
          secret-type: "generic"
          secret-name: ${{ env.AKS_SECRET_NAME_MI_CLIENTID }}
          string-data: '{ "${{ env.AKS_SECRET_NAME_MI_CLIENTID }}" : "${{ steps.get-managedIdentityClientId.outputs.managedIdentityClientId }}" }'
      - name: substitute tokens in deployment manifest
        uses: cschleiden/replace-tokens@v1.2
        with:
          tokenPrefix: "{"
          tokenSuffix: "}"
          files: ./src/ContosoTraders.Api.Products/Manifests/Deployment.yaml
        env:
          SUFFIX: ${{ vars.SUFFIX }}
          AKS_REPLICAS: ${{ env.AKS_REPLICAS }}
          AKS_CPU_LIMIT: ${{ env.AKS_CPU_LIMIT }}
          AKS_MEMORY_LIMIT: ${{ env.AKS_MEMORY_LIMIT }}
      - name: lint deployment manifest
        uses: azure/k8s-lint@v2.0
        with:
          manifests: ./src/ContosoTraders.Api.Products/Manifests/Deployment.yaml
      - name: apply deployment manifest
        uses: Azure/k8s-deploy@v4
        with:
          manifests: ./src/ContosoTraders.Api.Products/Manifests/Deployment.yaml
          images: ${{ env.ACR_NAME }}${{ vars.SUFFIX }}.azurecr.io/${{ env.PRODUCTS_ACR_REPOSITORY_NAME }}:${{ github.sha }}
          imagepullsecrets: ${{ env.AKS_SECRET_NAME_ACR_PASSWORD }}
          force: true
      - name: apply service manifest
        uses: Azure/k8s-deploy@v4
        with:
          pull-images: false
          manifests: ./src/ContosoTraders.Api.Products/Manifests/Service.yaml
          force: true
      # setup chaos mesh
      - name: apply namespace manifest (chaos-testing)
        uses: Azure/k8s-deploy@v4
        with:
          pull-images: false
          manifests: ./src/ContosoTraders.Api.Products/Manifests/NamespaceChaosTesting.yaml
          force: true
      - name: setup chaos mesh
        run: |
          az aks get-credentials --resource-group ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} --name ${{ env.AKS_CLUSTER_NAME }}${{ vars.SUFFIX }}
          ${{ steps.install-helm.outputs.helm-path }} repo add chaos-mesh https://charts.chaos-mesh.org
          ${{ steps.install-helm.outputs.helm-path }} repo update
          ${{ steps.install-helm.outputs.helm-path }} upgrade --install chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
      # create the ingress controller
      - name: create ingress controller
        run: |
          az aks get-credentials --resource-group ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} --name ${{ env.AKS_CLUSTER_NAME }}${{ vars.SUFFIX }}
          ${{ steps.install-helm.outputs.helm-path }} repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
          ${{ steps.install-helm.outputs.helm-path }} repo update
          ${{ steps.install-helm.outputs.helm-path }} upgrade --install --wait --timeout=1h nginx-ingress ingress-nginx/ingress-nginx \
            --set controller.replicaCount=1 \
            --set controller.nodeSelector."kubernetes\.io/os"=linux \
            --set defaultBackend.nodeSelector."kubernetes\.io/os"=linux \
            --set controller.admissionWebhooks.patch.nodeSelector."kubernetes\.io/os"=linux \
            --set controller.service.externalTrafficPolicy=Local
      - name: set dns label on public ip
        uses: azure/CLI@v1
        with:
          inlineScript: az network public-ip update --dns-name ${{ env.AKS_DNS_LABEL }}${{ vars.SUFFIX }} -g ${{ env.AKS_NODES_RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} -n $(az network public-ip list --query "[?starts_with(name,'kubernetes-') ].name" -o tsv -g ${{ env.AKS_NODES_RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }})
      # hack: extract the full fqdn / dns label of the aks app's public IP address
      - name: get aks-fqdn
        uses: azure/CLI@v1
        id: get-aks-fqdn
        with:
          # note: There should be a whitespace between ')' and ']'. More details: https://stackoverflow.com/a/59154958
          inlineScript: echo "aksFqdn"="$(az network public-ip list --query "[?starts_with(name,'kubernetes-') ].dnsSettings.fqdn" -o tsv -g ${{ env.AKS_NODES_RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }})" >> $GITHUB_OUTPUT
      # install cert-manager
      - name: apply namespace manifest (cert-manager)
        uses: Azure/k8s-deploy@v4
        with:
          pull-images: false
          manifests: ./src/ContosoTraders.Api.Products/Manifests/NamespaceCertManager.yaml
          force: true
      - name: install cert-manager
        run: |
          az aks get-credentials --resource-group ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} --name ${{ env.AKS_CLUSTER_NAME }}${{ vars.SUFFIX }}
          kubectl apply --validate=false -f https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.yaml
      - name: sleep for 30 seconds
        run: sleep 30s
        shell: bash
      - name: apply clusterIssuer manifest
        uses: Azure/k8s-deploy@v4
        with:
          pull-images: false
          manifests: ./src/ContosoTraders.Api.Products/Manifests/ClusterIssuer.yaml
          force: true
      - name: substitute tokens in certificate manifest
        uses: cschleiden/replace-tokens@v1.2
        with:
          tokenPrefix: "{"
          tokenSuffix: "}"
          files: ./src/ContosoTraders.Api.Products/Manifests/Certificate.yaml
        env:
          AKS_FQDN: ${{ steps.get-aks-fqdn.outputs.aksFqdn }}
      - name: apply certificate manifest
        uses: Azure/k8s-deploy@v4
        with:
          pull-images: false
          manifests: ./src/ContosoTraders.Api.Products/Manifests/Certificate.yaml
          force: true
      - name: substitute tokens in ingress manifest
        uses: cschleiden/replace-tokens@v1.2
        with:
          tokenPrefix: "{"
          tokenSuffix: "}"
          files: ./src/ContosoTraders.Api.Products/Manifests/Ingress.yaml
        env:
          AKS_FQDN: ${{ steps.get-aks-fqdn.outputs.aksFqdn }}
      - name: apply ingress manifest
        uses: Azure/k8s-deploy@v4
        with:
          pull-images: false
          manifests: ./src/ContosoTraders.Api.Products/Manifests/Ingress.yaml
          force: true
      - name: apply clusterRole manifest
        uses: Azure/k8s-deploy@v4
        with:
          pull-images: false
          manifests: ./src/ContosoTraders.Api.Products/Manifests/ClusterRole.yaml
          force: true
      - name: set productsApiEndpoint in kv
        uses: azure/CLI@v1
        with:
          inlineScript: az keyvault secret set --vault-name ${{ env.KV_NAME }}${{ vars.SUFFIX }} --name productsApiEndpoint --value ${{ steps.get-aks-fqdn.outputs.aksFqdn }} --description "endpoint url (fqdn) of the products api"
      - name: get products api endpoint
        uses: azure/CLI@v1
        id: get-productsApiEndpoint
        with:
          inlineScript: echo "productsApiEndpoint"="$(az keyvault secret show --vault-name ${{ env.KV_NAME }}${{ vars.SUFFIX }} --name productsApiEndpoint --query value -o tsv)" >> $GITHUB_OUTPUT

      #
      # section #4: deploy the ui
      #
      - name: set REACT_APP_APIURLSHOPPINGCART
        run: echo "REACT_APP_APIURLSHOPPINGCART"="https://${{ steps.get-cartsApiEndpoint.outputs.cartsApiEndpoint }}/v1" >> $GITHUB_ENV
      - name: set REACT_APP_APIURL
        run: echo "REACT_APP_APIURL"="https://${{ steps.get-productsApiEndpoint.outputs.productsApiEndpoint }}/v1" >> $GITHUB_ENV
      - name: set REACT_APP_B2CCLIENTID
        if: ${{ env.AADUSERNAME != '' && env.AADPASSWORD != '' }}
        run: echo "REACT_APP_B2CCLIENTID"="${{ steps.get-azureAdAppClientId.outputs.azureAdAppClientId }}" >> $GITHUB_ENV
      - uses: actions/setup-node@v3
        with:
          node-version: 18
          cache: npm
          cache-dependency-path: src/ContosoTraders.Ui.Website/package-lock.json
      - name: npm ci
        run: npm ci
        working-directory: src/ContosoTraders.Ui.Website
      - name: npm run build
        run: npm run build
        working-directory: src/ContosoTraders.Ui.Website
      - name: deploy ui to storage
        uses: azure/CLI@v1
        with:
          inlineScript: az storage blob sync --account-name '${{ env.UI_STORAGE_ACCOUNT_NAME }}${{ vars.SUFFIX }}' -c '$web' -s 'src/ContosoTraders.Ui.Website/build'
      - name: purge ui cdn endpoint
        uses: azure/CLI@v1
        with:
          inlineScript: az cdn endpoint purge --no-wait --content-paths '/*' -n '${{ env.UI_CDN_ENDPOINT_NAME }}${{ vars.SUFFIX }}' -g '${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }}' --profile-name '${{ env.CDN_PROFILE_NAME }}${{ vars.SUFFIX }}'
      - name: get ui cdn endpoint
        uses: azure/CLI@v1
        id: get-uiCdnEndpoint
        with:
          inlineScript: echo "uiCdnEndpoint"="$(az keyvault secret show --vault-name ${{ env.KV_NAME }}${{ vars.SUFFIX }} --name uiCdnEndpoint --query value -o tsv)" >> $GITHUB_OUTPUT
      - name: register auth callback (UI CDN)
        if: ${{ env.AADUSERNAME != '' && env.AADPASSWORD != '' }}
        uses: azure/CLI@v1
        with:
          inlineScript: |
            az rest \
              --method PATCH \
              --uri https://graph.microsoft.com/v1.0/applications/${{ steps.get-azureAdAppObjId.outputs.azureAdAppObjId }} \
              --headers 'Content-Type=application/json' \
              --body '{"spa":{"redirectUris":["https://localhost:3000/authcallback","http://localhost:3000/authcallback","https://staging.contosotraders.com/authcallback","https://production.contosotraders.com/authcallback","https://cloudtesting.contosotraders.com/authcallback","https://${{ steps.get-uiCdnEndpoint.outputs.uiCdnEndpoint }}/authcallback"]}}'
      - name: display ui cdn endpoint
        uses: azure/CLI@v1
        with:
          inlineScript: echo UI CDN endpoint accessible at https://$(az keyvault secret show --vault-name ${{ env.KV_NAME }}${{ vars.SUFFIX }} --name uiCdnEndpoint --query value -o tsv)

  load-tests-with-chaos-products-api:
    needs: [provision, playwright-tests-ui]
    runs-on: ubuntu-22.04
    concurrency:
      group: load-tests-with-chaos-products-api
      cancel-in-progress: true
    steps:
      - name: checkout code
        uses: actions/checkout@v4
      - name: azure login
        uses: azure/login@v1
        with:
          creds: ${{ secrets.SERVICEPRINCIPAL }}
      - name: get chaos experiment resource id
        uses: azure/CLI@v1
        id: get-chaosAksExperimentResourceId
        with:
          inlineScript: echo "chaosAksExperimentResourceId"="$(az resource show --resource-group ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }} --namespace Microsoft.Chaos --resource-type Experiments --name ${{ env.CHAOS_AKS_EXPERIMENT_NAME }}${{ vars.SUFFIX }} --query "id" -o tsv)" >> $GITHUB_OUTPUT
      - name: start chaos experiment (pod failure)
        uses: azure/CLI@v1
        with:
          inlineScript: az rest --method post --uri https://management.azure.com${{ steps.get-chaosAksExperimentResourceId.outputs.chaosAksExperimentResourceId }}/start?api-version=2021-09-15-preview
      - name: sleep for 30 seconds
        run: sleep 30s
        shell: bash
      - name: load test (products API)
        uses: Azure/load-testing@v1.1.19
        with:
          # Path of the YAML file. Should be fully qualified path or relative to the default working directory
          loadtestConfigFile: ./loadtests/contoso-traders-products.yaml
          loadtestResource: ${{ env.LOAD_TEST_SERVICE_NAME }}${{ vars.SUFFIX }}
          resourceGroup: ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }}
          env: |
            [
              {
                "name": "domain",
                "value": "${{ needs.provision.outputs.productsApiEndpoint }}"
              },
              {
                "name": "protocol",
                "value": "https"
              },
              {
                "name": "path",
                "value": "v1/Products/1"
              },
              {
                "name": "threads_per_engine",
                "value": "5"
              },
              {
                "name": "ramp_up_time",
                "value": "0"
              },
              {
                "name": "duration_in_sec",
                "value": "120"
              }
            ]

  load-tests-carts-internal-api:
    needs: [provision, playwright-tests-ui]
    runs-on: ubuntu-22.04
    concurrency:
      group: load-tests-carts-internal-api
      cancel-in-progress: true
    steps:
      - name: checkout code
        uses: actions/checkout@v4
      - name: azure login
        uses: azure/login@v1
        with:
          creds: ${{ secrets.SERVICEPRINCIPAL }}
      - name: get carts api endpoint (internal)
        uses: azure/CLI@v1
        id: get-cartsInternalApiEndpoint
        with:
          inlineScript: echo "cartsInternalApiEndpoint"="$(az keyvault secret show --vault-name ${{ env.KV_NAME }}${{ vars.SUFFIX }} --name cartsInternalApiEndpoint --query value -o tsv)" >> $GITHUB_OUTPUT
      - name: get vnetAcaSubnetId
        uses: azure/CLI@v1
        id: get-vnetAcaSubnetId
        with:
          inlineScript: echo "vnetAcaSubnetId"="$(az keyvault secret show --vault-name ${{ env.KV_NAME }}${{ vars.SUFFIX }} -n vnetAcaSubnetId --query "value" -o tsv)" >> $GITHUB_OUTPUT
      - name: substitute tokens in load test config file
        uses: cschleiden/replace-tokens@v1.2
        with:
          tokenPrefix: "{{"
          tokenSuffix: "}}"
          files: ./loadtests/contoso-traders-carts-internal.yaml
        env:
          LOAD_TEST_SUBNET_ID: ${{ steps.get-vnetAcaSubnetId.outputs.vnetAcaSubnetId }}
      - name: load test (carts internal API)
        uses: Azure/load-testing@v1.1.19
        with:
          # Path of the YAML file. Should be fully qualified path or relative to the default working directory
          loadtestConfigFile: ./loadtests/contoso-traders-carts-internal.yaml
          loadtestResource: ${{ env.LOAD_TEST_SERVICE_NAME }}${{ vars.SUFFIX }}
          resourceGroup: ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }}
          env: |
            [
              {
                "name": "domain",
                "value": "${{ steps.get-cartsInternalApiEndpoint.outputs.cartsInternalApiEndpoint }}"
              },
              {
                "name": "protocol",
                "value": "https"
              },
              {
                "name": "path",
                "value": "v1/ShoppingCart/loadtest"
              },
              {
                "name": "threads_per_engine",
                "value": "5"
              },
              {
                "name": "ramp_up_time",
                "value": "0"
              },
              {
                "name": "duration_in_sec",
                "value": "120"
              }
            ]

  load-tests-carts-api:
    needs: [provision, playwright-tests-ui]
    runs-on: ubuntu-22.04
    concurrency:
      group: load-tests-carts-api
      cancel-in-progress: true
    steps:
      - name: checkout code
        uses: actions/checkout@v4
      - name: azure login
        uses: azure/login@v1
        with:
          creds: ${{ secrets.SERVICEPRINCIPAL }}
      - name: load test (carts API)
        uses: Azure/load-testing@v1.1.19
        with:
          # Path of the YAML file. Should be fully qualified path or relative to the default working directory
          loadtestConfigFile: ./loadtests/contoso-traders-carts.yaml
          loadtestResource: ${{ env.LOAD_TEST_SERVICE_NAME }}${{ vars.SUFFIX }}
          resourceGroup: ${{ env.RESOURCE_GROUP_NAME }}${{ vars.SUFFIX }}
          env: |
            [
              {
                "name": "domain",
                "value": "${{ needs.provision.outputs.cartsApiEndpoint }}"
              },
              {
                "name": "protocol",
                "value": "https"
              },
              {
                "name": "path",
                "value": "v1/ShoppingCart/loadtest"
              },
              {
                "name": "threads_per_engine",
                "value": "5"
              },
              {
                "name": "ramp_up_time",
                "value": "0"
              },
              {
                "name": "duration_in_sec",
                "value": "120"
              }
            ]

  playwright-tests-ui:
    needs: [provision]
    timeout-minutes: 20
    runs-on: ubuntu-22.04
    container:
      image: mcr.microsoft.com/playwright:v1.36.2-jammy
    defaults:
      run:
        working-directory: src/ContosoTraders.Ui.Website
    env:
      AADUSERNAME: ${{ secrets.AADUSERNAME }}
      AADPASSWORD: ${{ secrets.AADPASSWORD }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v3
        with:
          node-version: 18
          cache: npm
          cache-dependency-path: src/ContosoTraders.Ui.Website/package-lock.json
      - name: Set env variables for testing endpoints
        run: |
          echo "REACT_APP_APIURLSHOPPINGCART"="https://${{ needs.provision.outputs.cartsApiEndpoint }}/v1" >> $GITHUB_ENV
          echo "REACT_APP_APIURL"="https://${{ needs.provision.outputs.productsApiEndpoint }}/v1" >> $GITHUB_ENV
          echo "REACT_APP_BASEURLFORPLAYWRIGHTTESTING"="https://${{ needs.provision.outputs.uiCdnEndpoint }}" >> $GITHUB_ENV
      - name: Set env variables for testing login
        if: ${{ env.AADUSERNAME != '' && env.AADPASSWORD != '' }}
        run: |
          echo "REACT_APP_B2CCLIENTID"="${{ needs.provision.outputs.azureAdAppClientId }}" >> $GITHUB_ENV
          echo "REACT_APP_AADUSERNAME"="${{ env.AADUSERNAME }}" >> $GITHUB_ENV
          echo "REACT_APP_AADPASSWORD"="${{ env.AADPASSWORD }}" >> $GITHUB_ENV
      - name: install dependencies
        run: npm ci
      - name: run playwright tests
        id: test
        run: HOME=/root npx playwright test
      - name: upload playwright report
        uses: actions/upload-artifact@v3
        # only upload report if tests failed
        if: steps.test.outcome != 'success'
        with:
          name: playwright-report
          path: src/ContosoTraders.Ui.Website/playwright-report/
          retention-days: 30