azure-pipelines.yml

# These jobs are triggered automatically and they test code, examples, and wheels.
# Additional checks can be manually triggered
variables:
  nightlyRelease: $[and(eq(variables['nightly_check'], true), eq(variables['Build.SourceBranch'], 'refs/heads/master'), ne(variables['Build.Reason'], 'PullRequest'))]

trigger:
- master

pr:
- master

jobs:

- job: 'manylinux2014'
  pool:
    vmImage: 'ubuntu-latest'
  strategy:
    matrix:
      Python37:
        arch: x86_64
        plat: manylinux2014_x86_64
        python_ver: '37'
        python.version: '3.7'
      Python38:
        arch: x86_64
        plat: manylinux2014_x86_64
        python_ver: '38'
        python.version: '3.8'
      Python39:
        arch: x86_64
        plat: manylinux2014_x86_64
        python_ver: '39'
        python.version: '3.9'
      Python310:
        arch: x86_64
        plat: manylinux2014_x86_64
        python_ver: '310'
        python.version: '3.10'
  variables:
    CCACHE_DIR: $(Pipeline.Workspace)/ccache

  steps:
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version)'

  - task: Cache@2
    inputs:
      key: '"ccache-wheels-v2022.8.25" | $(Agent.OS) | "$(python.version)"'
      path: $(CCACHE_DIR)
    displayName: ccache

  - bash: |
      set -e
      sed -i "s/'giotto-tda'/'giotto-tda-nightly'/1" setup.py
      sed -i 's/"giotto-tda"/"giotto-tda-nightly"/1' setup.py
      sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py
    condition: eq(variables.nightlyRelease, true)
    displayName: 'Change name to giotto-tda-nightly'

  - task: Bash@3
    inputs:
      filePath: .azure-ci/build_manylinux2014.sh
    env:
      python_ver: $(python_ver)
      CCACHE_DIR: $(CCACHE_DIR)
    displayName: 'Run docker container, install and uninstall dev environment, test with pytest and flake8, build the wheels'

  - script: |
      set -e
      python -m pip install --upgrade pip
      python -m pip install dist/*manylinux2014*.whl
    displayName: 'Install the wheels'

  - script: |
      set -e
      python -m pip install pandas pytest pytest-cov pytest-azurepipelines pytest-benchmark hypothesis
      # nbformat is needed by plotly: https://github.com/plotly/plotly.py/issues/2159
      python -m pip install nbformat
      mkdir tmp_test_cov
      cd tmp_test_cov
      pytest --pyargs gtda --ignore-glob='*externals*' --no-cov --no-coverage-upload
    condition: eq(variables['test_wheels'], 'true')
    displayName: 'Test the wheels with pytest'

  - script: |
      set -e
      python -m pip install openml matplotlib umap-learn gensim
      python -m pip install papermill
      cd examples
      for n in *.ipynb
      do
        papermill --start_timeout 2000 $n -
      done
    condition: eq(variables['test_notebooks'], 'true')
    displayName: 'Test jupyter notebooks with papermill'

  - task: CopyFiles@2
    displayName: 'Copy files'
    inputs:
      contents: 'dist/*'
      targetFolder: '$(Build.ArtifactStagingDirectory)'

  - task: PublishBuildArtifacts@1
    displayName: 'Create download link'
    inputs:
      pathToPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: 'wheel'

  - bash: |
      set -e
      python -m pip install twine
      twine upload -u giotto-learn -p $(pypi_psw) --skip-existing dist/*manylinux2014*.whl
    condition: eq(variables.nightlyRelease, true)
    displayName: 'Upload nightly wheels to PyPI'


- job: 'macOS11'
  pool:
    vmImage: 'macOS-11'
  strategy:
    matrix:
      Python37:
        python.version: '3.7'
      Python38:
        python.version: '3.8'
      Python39:
        python.version: '3.9'
      Python310:
        python.version: '3.10'
  variables:
    CCACHE_DIR: $(Pipeline.Workspace)/ccache

  steps:
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version)'

  - bash: |
      set -e
      sed -i.bak "s/'giotto-tda'/'giotto-tda-nightly'/1" setup.py
      sed -i.bak 's/"giotto-tda"/"giotto-tda-nightly"/1' setup.py
      rm setup.py.bak
      sed -i.bak "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py
      rm gtda/_version.py.bak
    condition: eq(variables.nightlyRelease, true)
    displayName: 'Change name to giotto-tda-nightly'

  - task: Cache@2
    inputs:
      key: '"ccache-v2022.8.25" | $(Agent.OS) | "$(python.version)"'
      path: $(CCACHE_DIR)
    displayName: ccache

  # Python and gcc hotfixes following https://github.com/actions/virtual-environments/issues/2322 and https://github.com/actions/virtual-environments/issues/2391
  - script: |
      set -e      
      brew install boost ccache
    displayName: 'Install system dependencies'

  - script: |
      set -e
      python -m pip install --upgrade pip setuptools
      python -m pip install wheel
      source .azure-ci/setup_ccache.sh
      python -m pip install -e ".[dev]"
      ccache -s
    displayName: 'Install dependencies and dev environment'

  - script: |
      set -e
      pytest gtda --cov --cov-report xml
    displayName: 'Test dev install with pytest, upload coverage report'

  - script: |
      set -e
      flake8
    displayName: 'Test dev install with flake8'

  - script: |
      set -e
      python -m pip uninstall -y giotto-tda
      python -m pip uninstall -y giotto-tda-nightly
    displayName: 'Uninstall giotto-tda/giotto-tda-nightly'

  # Includes workaround for macOS 11 compatibility issue, see https://github.com/apple/tensorflow_macos/issues/46
  - script: |
      set -e
      python setup.py bdist_wheel
      for f in dist/*.whl; do mv "$f" "${f/_11_0_/_10_16_}"; done
    displayName: 'Build the wheels'

  - script: python -m pip install dist/*.whl
    displayName: 'Install the wheels'

  - script: |
      set -e
      mkdir tmp_test_cov
      cd tmp_test_cov
      pytest --pyargs gtda --ignore-glob='*externals*' --no-cov --no-coverage-upload
    condition: eq(variables['test_wheels'], 'true')
    displayName: 'Test the wheels with pytest'

  - script: |
      set -e
      python -m pip install -e ".[examples]"
      python -m pip install papermill
      cd examples
      for n in *.ipynb
      do
        papermill --start_timeout 2000 $n -
      done
    condition: eq(variables['test_notebooks'], 'true')
    displayName: 'Test jupyter notebooks with papermill'

  - task: CopyFiles@2
    displayName: 'Copy files'
    inputs:
      contents: 'dist/*'
      targetFolder: '$(Build.ArtifactStagingDirectory)'

  - script: |
      set -e
      python -m pip install twine
      twine check dist/*
    displayName: 'Check distribution with twine'

  - task: PublishBuildArtifacts@1
    displayName: 'Create download link'
    inputs:
      pathToPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: 'wheel'

  - bash: |
      set -e
      twine upload -u giotto-learn -p $(pypi_psw) --skip-existing dist/*
    condition: eq(variables.nightlyRelease, true)
    displayName: 'Upload nightly wheels to PyPI'


- job: 'windows'
  pool:
    vmImage: 'windows-latest'
  strategy:
    matrix:
      Python37:
        python_ver: '37'
        python.version: '3.7'
      Python38:
        python_ver: '38'
        python.version: '3.8'
      Python39:
        python.version: '3.9'
      Python310:
        python.version: '3.10'

  steps:
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '$(python.version)'

  - bash: |
      set -e
      sed -i "s/'giotto-tda'/'giotto-tda-nightly'/1" setup.py
      sed -i 's/"giotto-tda"/"giotto-tda-nightly"/1' setup.py
      sed -i "s/__version__.*/__version__ = '$(Build.BuildNumber)'/1" gtda/_version.py
    condition: eq(variables.nightlyRelease, true)
    displayName: 'Change name to giotto-tda-nightly'

  # Use the boost_1_72_0-msvc-14.1-64.exe for windows-latest
  # Following issue https://github.com/actions/virtual-environments/issues/2667
  # As of March 2021, it is necessary to download Boost manually
  # DIR: installation dir, this value need to be forwarded to BOOST_ROOT_PIPELINE
  - powershell: |
      $Url = "https://sourceforge.net/projects/boost/files/boost-binaries/1.72.0/boost_1_72_0-msvc-14.2-64.exe"
      (New-Object System.Net.WebClient).DownloadFile($Url, "$env:TEMP\boost.exe")
      Start-Process -Wait -FilePath "$env:TEMP\boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=C:\hostedtoolcache\windows\Boost\1.72.0\x86_64"
    displayName: 'Install boost'

  # Set BOOST_ROOT_PIPELINE to the version used in the pipeline
  # See https://github.com/actions/virtual-environments/issues/687#issuecomment-616345933
  - script: |
      echo "##vso[task.setvariable variable=BOOST_ROOT_PIPELINE]C:\hostedtoolcache\windows\Boost\1.72.0\x86_64"
    displayName: 'Set env variable for boost version'

  - script: |
      python -m pip install --upgrade pip setuptools
      python -m pip install wheel
      python -m pip install -e ".[dev]"
    displayName: 'Install dev environment'

  - script: |
      pytest gtda --no-cov --no-coverage-upload || exit /b
    displayName: 'Test dev install with pytest'

  - script: |
      python -m pip uninstall -y giotto-tda
      python -m pip uninstall -y giotto-tda-nightly
    displayName: 'Uninstall giotto-tda/giotto-tda-nightly'

  - bash: |
      set -e
      sed -i $'s/\r$//' README.rst
      python setup.py bdist_wheel
    displayName: 'Build the wheels'

  - bash: python -m pip install dist/*.whl
    displayName: 'Install the wheels'

  - script: |
      mkdir tmp_test_cov
      cd tmp_test_cov
      pytest --pyargs gtda --ignore-glob='*externals*' --no-cov --no-coverage-upload
    condition: eq(variables['test_wheels'], 'true')
    displayName: 'Test the wheels with pytest'

  - script: |
      python -m pip install -e ".[examples]"
      python -m pip install papermill
      cd examples
      FOR %%n in (*.ipynb) DO (papermill --start_timeout 2000 %%n - || exit /b)
    condition: eq(variables['test_notebooks'], 'true')
    displayName: 'Test jupyter notebooks with papermill'

  - task: CopyFiles@2
    displayName: 'Copy files'
    inputs:
      contents: 'dist/*'
      targetFolder: '$(Build.ArtifactStagingDirectory)'

  - task: PublishBuildArtifacts@1
    displayName: 'Create download link'
    inputs:
      pathToPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: 'wheel'

  - bash: |
      set -e
      python -m pip install twine
      twine upload -u giotto-learn -p $(pypi_psw) --skip-existing dist/*
    condition: eq(variables.nightlyRelease, true)
    displayName: 'Upload nightly wheels to PyPI'