.gitlab-ci.yml

include:
  - project: 'analytics/artificial-intelligence/ai-platform/aip-infrastructure/ci-templates/ci-cd-template'
#    ref: &include_ref 'v4'
    ref: &include_ref '4e25c7200369aec17fcf8de723db360a7726151a'
    file: 'environments/devex.yml'
  - project: 'analytics/artificial-intelligence/ai-platform/aip-infrastructure/ci-templates/ci-cd-template'
    ref: *include_ref
    file: '/blocks/python.yml'

variables:
  PY_LIBRARY_NAME: "zillow-metaflow"
  MAJOR_VERSION: "2"
  MINOR_VERSION: "1"
  METAFLOW_VERSION_PATH: "setup.py"
  IMAGE_REPOSITORY_TAG_PATH: 'image_tag_file.txt'
  IMAGE_REPOSITORY_TAG_PATH_AIP_STEP: 'image_tag_file_aip_step.txt'
  # Run on the Internal cluster only on commit.
  # We run the tests via the integration test framework for Non-Prod/Prod.
  DEPLOY_INTERNAL: "true"
  DEPLOY_STAGE: "false"
  DEPLOY_PROD: "false"
  DEPLOY_SANDBOX: "false"
  PUBLISH: "true"
  CI_TRACE_DEBUG: "true"

stages:
  - build
  - test

.version: &version |
  # Extract the open source Metaflow version as the basis for our forked library version.
  METAFLOW_VERSION=$(cat $METAFLOW_VERSION_PATH | sed -nr "s/^version = ['\"]([^'\"]*)['\"]$/\1/p")

  PY_LIBRARY_VERSION="${MAJOR_VERSION}.${MINOR_VERSION}.${CI_PIPELINE_IID}"
  if [ "${CI_COMMIT_BRANCH}" != "${CI_DEFAULT_BRANCH}" ]; then
    PY_LIBRARY_VERSION="${PY_LIBRARY_VERSION}-dev.${CI_PIPELINE_IID}"
    # Ensure the image tag is compliant with PEP-440,
    IMAGE_TAG="${PY_LIBRARY_VERSION}.dev${CI_PIPELINE_IID}"
  else
    IMAGE_TAG="${PY_LIBRARY_VERSION}"
  fi
  # Only apply the Metaflow version to the python library as Docker versions do not have the concept of
  # build metadata and the "+" character causes errors.
  PY_LIBRARY_VERSION="${PY_LIBRARY_VERSION}+${METAFLOW_VERSION}"

.test:
  extends: .generate_kubeconfig
  stage: test
  variables:
    GIT_STRATEGY: none
    PIPELINE_VERSION: "1.0"
  before_script:
    - export IMAGE_REPOSITORY_TAG=$( cat ${IMAGE_REPOSITORY_TAG_PATH} )
    - export IMAGE_REPOSITORY_TAG_AIP_STEP=$( cat ${IMAGE_REPOSITORY_TAG_PATH_AIP_STEP} )
    - !reference [.generate_kubeconfig, before_script]
  script:
    - docker run
        --rm
        -v ${SHARED_PATH}/.kube:/home/zservice/.kube
        -v ${SHARED_PATH}/public:/home/zservice/public  
        -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID 
        -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY 
        -e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN 
        -e AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION 
        -e ARGO_RUN_URL_PREFIX=$ARGO_RUN_URL_PREFIX
        -e METAFLOW_RUN_URL_PREFIX=$METAFLOW_RUN_URL_PREFIX 
        -e METAFLOW_KUBERNETES_NAMESPACE=$METAFLOW_KUBERNETES_NAMESPACE 
        -e USER=$GITLAB_USER_EMAIL
        -e AIP_STEP_IMAGE=${IMAGE_REPOSITORY_TAG_AIP_STEP}
        ${IMAGE_REPOSITORY_TAG}
        bash -c "
          set -x &&
          cd /home/zservice/metaflow/metaflow/plugins/aip/tests &&
          python -m pytest -s -n 7 run_integration_tests.py --image ${IMAGE_REPOSITORY_TAG} --opsgenie-api-token ${OPSGENIE_API_TOKEN} --cov-config=setup.cfg --pipeline-tag pipeline_iid_${CI_PIPELINE_IID}
        "
  artifacts:
    when: always
    paths:
      - public

build:docker:
  stage: build
  script:
    - *version
    - |
      build_image() {
        image_repository_tag=$1
        image_repository_tag_path=$2

        DOCKER_BUILDKIT=1 docker build \
          --no-cache -f metaflow/plugins/aip/tests/Dockerfile \
          --build-arg IMAGE_TAG=${image_repository_tag} \
          -t ${image_repository_tag} .
        echo ${DOCKER_API_KEY} | docker login -u ${DOCKER_USERNAME} --password-stdin ${DOCKER_REPO_URL}
        STDERR=$(docker push ${image_repository_tag} 2>&1 >/dev/null) || true
        echo $STDERR

        if [ -z "$STDERR" ] || [[ $STDERR == *"manifest invalid"* ]] || [[ $STDERR == *"Not enough permissions to delete/overwrite artifact"* ]]; then
          echo "Successful!"
        else
          echo "Error in pushing image to Artifactory." >&2
          exit 1
        fi
        echo ${image_repository_tag} > ${image_repository_tag_path}
      }

    - |
      export IMAGE_REPOSITORY_TAG=${DOCKER_REPO_URL}/${CI_PROJECT_NAMESPACE}/${PY_LIBRARY_NAME}:${IMAGE_TAG}
      build_image $IMAGE_REPOSITORY_TAG $IMAGE_REPOSITORY_TAG_PATH

    # Create another image specifically to test aip_flow.py, this sets the image name within the built image
    # such that the test can assert that for specific steps it is using this overridden image
    - |
      export IMAGE_REPOSITORY_TAG_AIP_STEP=${DOCKER_REPO_URL}/${CI_PROJECT_NAMESPACE}/${PY_LIBRARY_NAME}:${IMAGE_TAG}_aip_step
      build_image $IMAGE_REPOSITORY_TAG_AIP_STEP $IMAGE_REPOSITORY_TAG_PATH_AIP_STEP
  artifacts:
    paths:
      - ${IMAGE_REPOSITORY_TAG_PATH}
      - ${IMAGE_REPOSITORY_TAG_PATH_AIP_STEP}

build:publish:
  extends: .aip_python_debian_image
  stage: build
  script:
  - *version
  # Now write back the zillow-metaflow version back to the original location we found the original so
  # python package managers can reference it as they need the version stored internally.
  - sed -i "s/\(version = ['\"]\)[^'\"]*\(['\"]\)/\1${PY_LIBRARY_VERSION}\2/" $METAFLOW_VERSION_PATH
  - python setup.py sdist
  # Set up the configuration for Artifactory to publish the python package internally.
  - |
    cat >~/.pypirc <<EOL
    [distutils]
    index-servers = local
    [local]
    repository: ${ANALYTICS_PYPI_REPOSITORY}
    username: ${PYPI_USERNAME}
    password: ${PYPI_PASSWORD}
    EOL
  - python setup.py sdist upload --repository "${ANALYTICS_PYPI_REPOSITORY}"
  rules:
    - if: '$PUBLISH == "true"'

test:sandbox:
  extends: 
    - .devex_sandbox_eks
    - .test
  variables:
    ARGO_RUN_URL_PREFIX: "https://argo-server.int.sandbox-k8s.zg-aip.net/"
    METAFLOW_RUN_URL_PREFIX: "https://metaflow.int.sandbox-k8s.zg-aip.net"
    METAFLOW_KUBERNETES_NAMESPACE: "metaflow-integration-testing-sandbox"
  rules:
    - if: '$DEPLOY_SANDBOX == "true"'

test:internal:
  extends: 
    - .devex_internal_eks
    - .test
  variables:
    ARGO_RUN_URL_PREFIX: "https://argo-server.int.dev-k8s.zg-aip.net/"
    METAFLOW_RUN_URL_PREFIX: "https://metaflow.int.dev-k8s.zg-aip.net"
    METAFLOW_KUBERNETES_NAMESPACE: "metaflow-integration-testing-internal"
  rules:
    - if: '$DEPLOY_INTERNAL == "true"'

test:nonprod:
  extends: 
    - .devex_nonprod_eks
    - .test
  variables:
    ARGO_RUN_URL_PREFIX: "https://argo-server.int.stage-k8s.zg-aip.net/"
    METAFLOW_RUN_URL_PREFIX: "https://metaflow.int.stage-k8s.zg-aip.net"
    METAFLOW_KUBERNETES_NAMESPACE: "metaflow-integration-testing-stage"
  rules:
    - if: '$DEPLOY_STAGE == "true"'

test:prod:
  extends: 
  - .devex_prod_eks
  - .test
  variables:
    ARGO_RUN_URL_PREFIX: "https://argo-server.int.prod-k8s.zg-aip.net/"
    METAFLOW_RUN_URL_PREFIX: "https://metaflow.int.prod-k8s.zg-aip.net"
    METAFLOW_KUBERNETES_NAMESPACE: "metaflow-integration-testing-prod"
  rules:
    - if: '$DEPLOY_PROD == "true"'