remove .numpy which failed in gpu/distributed env #1814
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: tfra-release | |
| on: | |
| release: | |
| types: [published] | |
| tags: | |
| - v* | |
| push: | |
| branches: | |
| - master | |
| - r* | |
| pull_request: | |
| branches: | |
| - master | |
| - r* | |
| env: | |
| MIN_PY_VERSION: '3.7' | |
| MAX_PY_VERSION: '3.11' | |
| HOROVOD_VERSION: '0.28.1' | |
| jobs: | |
| test-with-bazel: | |
| name: Test with bazel | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ env.MAX_PY_VERSION }} | |
| - name: Build wheels | |
| run: | | |
| pip install --default-timeout=1000 -r tools/install_deps/pytest.txt -r tools/install_deps/tensorflow.txt -r requirements.txt | |
| sudo apt install -y redis > /dev/null 2> /dev/null | |
| bash tools/install_deps/install_bazelisk.sh ./ | |
| python -m pip install tensorflow-io | |
| python -m pip install --upgrade protobuf==4.23.4 | |
| python configure.py | |
| bazel test --local_ram_resources=4096 -c opt -k --test_timeout 300,450,1200,3600 --test_output=errors //tensorflow_recommenders_addons/... | |
| release-wheel-cpu: | |
| name: Build release wheels for CPU | |
| strategy: | |
| matrix: | |
| # TODO: add back 'windows-latest' when it can be compiled. | |
| os: ['macos-12', 'ubuntu-20.04'] | |
| py-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] | |
| tf-version: ['2.11.0', '2.15.1'] | |
| tf-need-cuda: ['0'] | |
| cpu: ['x86'] | |
| exclude: | |
| # excludes cuda on macOS | |
| - tf-version: '2.11.0' | |
| py-version: '3.11' | |
| - tf-version: '2.15.1' | |
| py-version: '3.7' | |
| - tf-version: '2.15.1' | |
| py-version: '3.8' | |
| fail-fast: false | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: clear cache folder | |
| run: rm -rf /usr/share/dotnet /opt/ghc "/usr/local/share/boost" | |
| - uses: actions/github-script@0.3.0 | |
| id: author-date | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| const commit_details = await github.git.getCommit({owner: context.repo.owner, repo: context.repo.repo, commit_sha: context.sha}); | |
| return commit_details.data.author.date | |
| - if: matrix.tf-version != '2.15.1' | |
| shell: bash | |
| run: echo "SKIP_CUSTOM_OP_TESTS=--skip-custom-ops" >> $GITHUB_ENV | |
| - if: github.event_name == 'push' | |
| shell: bash | |
| run: echo "NIGHTLY_FLAG=--nightly" >> $GITHUB_ENV | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.py-version }} | |
| - name: Setup Bazel | |
| # Ubuntu bazel is run inside of the docker image | |
| if: matrix.os != 'ubuntu-20.04' | |
| run: bash tools/install_deps/install_bazelisk.sh ./ | |
| - name: Build wheels | |
| env: | |
| OS: ${{ runner.os }} | |
| PY_VERSION: ${{ matrix.py-version }} | |
| TF_VERSION: ${{ matrix.tf-version }} | |
| TF_NEED_CUDA: ${{ matrix.tf-need-cuda }} | |
| NIGHTLY_TIME: ${{ steps.author-date.outputs.result }} | |
| CPU: ${{ matrix.cpu }} | |
| shell: bash | |
| run: | | |
| if [[ "$TF_VERSION" =~ ^2\.(11|12|13|14|15)\.[0-9]$ ]] ; then | |
| export HOROVOD_VERSION="0.28.1" | |
| fi | |
| bash .github/workflows/make_wheel_${OS}_${CPU}.sh | |
| - uses: haya14busa/action-cond@v1 | |
| id: device | |
| with: | |
| cond: ${{ matrix.tf-need-cuda == '1' }} | |
| if_true: "gpu" | |
| if_false: "cpu" | |
| - uses: actions/upload-artifact@v1 | |
| with: | |
| name: ${{ runner.os }}-${{ matrix.py-version }}-tf${{ matrix.tf-version }}-${{ steps.device.outputs.value }}-${{ matrix.cpu }}-wheel | |
| path: wheelhouse | |
| release-wheel-gpu: | |
| name: Build release wheels for GPU | |
| needs: [release-wheel-cpu, test-with-bazel] | |
| strategy: | |
| matrix: | |
| # TODO: add back 'windows-latest' when it can be compiled. | |
| os: ['self-hosted-gpu'] | |
| py-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] | |
| tf-version: ['2.11.0', '2.15.1'] | |
| tf-need-cuda: ['1'] | |
| cpu: ['x86'] | |
| exclude: | |
| - tf-version: '2.11.0' | |
| py-version: '3.11' | |
| - tf-version: '2.15.1' | |
| py-version: '3.7' | |
| - tf-version: '2.15.1' | |
| py-version: '3.8' | |
| fail-fast: false | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: clear cache folder | |
| run: rm -rf /usr/share/dotnet /opt/ghc "/usr/local/share/boost" | |
| - uses: actions/github-script@0.3.0 | |
| id: author-date | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| const commit_details = await github.git.getCommit({owner: context.repo.owner, repo: context.repo.repo, commit_sha: context.sha}); | |
| return commit_details.data.author.date | |
| - if: matrix.tf-version != '2.15.1' | |
| shell: bash | |
| run: echo "SKIP_CUSTOM_OP_TESTS=--skip-custom-ops" >> $GITHUB_ENV | |
| - if: github.event_name == 'push' | |
| shell: bash | |
| run: echo "NIGHTLY_FLAG=--nightly" >> $GITHUB_ENV | |
| - uses: actions/checkout@v2 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.py-version }} | |
| - name: Setup Bazel | |
| # Ubuntu bazel is run inside of the docker image | |
| if: matrix.os != 'ubuntu-20.04' | |
| run: bash tools/install_deps/install_bazelisk.sh ./ | |
| - name: Build wheels | |
| env: | |
| OS: ${{ runner.os }} | |
| PY_VERSION: ${{ matrix.py-version }} | |
| TF_VERSION: ${{ matrix.tf-version }} | |
| TF_NEED_CUDA: ${{ matrix.tf-need-cuda }} | |
| NIGHTLY_TIME: ${{ steps.author-date.outputs.result }} | |
| CPU: ${{ matrix.cpu }} | |
| shell: bash | |
| run: | | |
| if [[ "$TF_VERSION" =~ ^2\.(11|12|13|14|15)\.[0-9]$ ]] ; then | |
| export HOROVOD_VERSION="0.28.1" | |
| fi | |
| bash .github/workflows/make_wheel_${OS}_${CPU}.sh | |
| - uses: haya14busa/action-cond@v1 | |
| id: device | |
| with: | |
| cond: ${{ matrix.tf-need-cuda == '1' }} | |
| if_true: "gpu" | |
| if_false: "cpu" | |
| - uses: actions/upload-artifact@v1 | |
| with: | |
| name: ${{ runner.os }}-${{ matrix.py-version }}-tf${{ matrix.tf-version }}-${{ steps.device.outputs.value }}-${{ matrix.cpu }}-wheel | |
| path: wheelhouse | |
| upload-wheels: | |
| name: Publish wheels to PyPi | |
| needs: [release-wheel-gpu] | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| matrix: | |
| # TODO: add back 'Windows' when it can be compiled. | |
| os: ['macOS', 'Linux'] | |
| py-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] | |
| tf-version: ['2.15.1'] | |
| tf-need-cuda: ['0', '1'] | |
| cpu: ['x86'] | |
| exclude: | |
| # excludes cuda on macOS | |
| - os: 'macOS' | |
| tf-need-cuda: '1' | |
| - tf-version: '2.11.0' | |
| py-version: '3.11' | |
| - tf-version: '2.15.1' | |
| py-version: '3.7' | |
| - tf-version: '2.15.1' | |
| py-version: '3.8' | |
| fail-fast: false | |
| if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || github.event_name == 'release' | |
| steps: | |
| - uses: haya14busa/action-cond@v1 | |
| id: device | |
| with: | |
| cond: ${{ matrix.tf-need-cuda == '1' }} | |
| if_true: "gpu" | |
| if_false: "cpu" | |
| - uses: actions/download-artifact@v1 | |
| with: | |
| name: ${{ matrix.os }}-${{ matrix.py-version }}-tf${{ matrix.tf-version }}-${{ steps.device.outputs.value }}-${{ matrix.cpu }}-wheel | |
| path: ./dist | |
| - run: | | |
| set -e -x | |
| ls -la dist/ | |
| sha256sum dist/*.whl | |
| - uses: pypa/gh-action-pypi-publish@v1.1.0 | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_API_TOKEN }} | |
| upload-dev-container: | |
| name: Upload dev container to DockerHub | |
| needs: [release-wheel-gpu, test-with-bazel] | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| matrix: | |
| py-version: ['3.10'] | |
| if: (github.event_name == 'push' && github.ref == 'refs/heads/master') | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - run: | | |
| set -e -x | |
| echo ${{ secrets.DOCKER_PW }} | docker login --username ${{ secrets.DOCKER_USER }} --password-stdin | |
| export PY_VERSION=${{ matrix.py-version }} | |
| bash .github/workflows/github_build_dev_container.sh | |
| docker push tfra/dev_container:latest-python${{ matrix.py-version }} |