Add notebook documentation for deploying PySyft #7039
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR Tests - Stack | |
| on: | |
| # schedule: | |
| # - cron: "0 22 * * *" # 7pm Brazil, 10pm UTC, 8am AEST | |
| workflow_call: | |
| pull_request: | |
| branches: | |
| - dev | |
| - main | |
| - "0.8" | |
| workflow_dispatch: | |
| inputs: | |
| none: | |
| description: "Run Version Tests Manually" | |
| required: false | |
| concurrency: | |
| group: stack-${{ github.event_name == 'pull_request' && format('{0}-{1}', github.workflow, github.event.pull_request.number) || github.workflow_ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| pr-syft-image-test: | |
| strategy: | |
| max-parallel: 99 | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ["3.12"] | |
| fail-fast: false | |
| runs-on: ${{matrix.os}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check for file changes | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| base: ${{ github.ref }} | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| if: steps.changes.outputs.stack == 'true' | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Upgrade pip | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| pip install --upgrade pip uv==0.1.35 | |
| uv --version | |
| - name: Get pip cache dir | |
| if: steps.changes.outputs.stack == 'true' | |
| id: pip-cache | |
| shell: bash | |
| run: | | |
| echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT | |
| - name: pip cache | |
| uses: actions/cache@v4 | |
| if: steps.changes.outputs.stack == 'true' | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-uv-py${{ matrix.python-version }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-py${{ matrix.python-version }} | |
| - name: Install tox | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| pip install --upgrade tox tox-uv==1.5.1 | |
| - name: Run syft backend base image building test | |
| if: steps.changes.outputs.stack == 'true' | |
| timeout-minutes: 60 | |
| run: | | |
| tox -e backend.test.basecpu | |
| pr-tests-syft-integration: | |
| strategy: | |
| max-parallel: 99 | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ["3.12"] | |
| pytest-modules: ["local_node"] | |
| fail-fast: false | |
| runs-on: ${{matrix.os}} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check for file changes | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| base: ${{ github.ref }} | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| if: steps.changes.outputs.stack == 'true' | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Upgrade pip | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| pip install --upgrade pip uv==0.1.35 | |
| uv --version | |
| - name: Get pip cache dir | |
| if: steps.changes.outputs.stack == 'true' | |
| id: pip-cache | |
| shell: bash | |
| run: | | |
| echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT | |
| - name: pip cache | |
| uses: actions/cache@v4 | |
| if: steps.changes.outputs.stack == 'true' | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-uv-py${{ matrix.python-version }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-py${{ matrix.python-version }} | |
| - name: Install tox | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| pip install --upgrade tox tox-uv==1.5.1 | |
| - name: Run Syft Integration Tests | |
| if: steps.changes.outputs.stack == 'true' | |
| timeout-minutes: 60 | |
| env: | |
| PYTEST_MODULES: "${{ matrix.pytest-modules }}" | |
| GITHUB_CI: true | |
| shell: bash | |
| run: | | |
| tox -e syft.test.integration | |
| pr-tests-integration-k8s: | |
| strategy: | |
| max-parallel: 99 | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ["3.12"] | |
| pytest-modules: ["frontend network container_workload"] | |
| fail-fast: false | |
| runs-on: ${{matrix.os}} | |
| steps: | |
| - name: Permission to home directory | |
| run: | | |
| sudo chown -R $USER:$USER $HOME | |
| - uses: actions/checkout@v4 | |
| - name: Check for file changes | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| base: ${{ github.ref }} | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| if: steps.changes.outputs.stack == 'true' | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Add K3d Registry | |
| run: | | |
| sudo python ./scripts/patch_hosts.py --add-k3d-registry | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: true | |
| large-packages: false | |
| # free 10GB of space | |
| - name: Remove unnecessary files | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| docker image prune --all --force | |
| docker builder prune --all --force | |
| docker system prune --all --force | |
| - name: Upgrade pip | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| pip install --upgrade pip uv==0.1.35 | |
| uv --version | |
| - name: Get pip cache dir | |
| if: steps.changes.outputs.stack == 'true' | |
| id: pip-cache | |
| shell: bash | |
| run: | | |
| echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT | |
| - name: pip cache | |
| uses: actions/cache@v4 | |
| if: steps.changes.outputs.stack == 'true' | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-uv-py${{ matrix.python-version }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-py${{ matrix.python-version }} | |
| - name: Install tox | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| pip install --upgrade tox tox-uv==1.5.1 | |
| - name: Install kubectl | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| # cleanup apt version | |
| sudo apt remove kubectl || true | |
| # install kubectl 1.27 | |
| curl -LO https://dl.k8s.io/release/v1.27.2/bin/linux/amd64/kubectl | |
| chmod +x kubectl | |
| sudo install kubectl /usr/local/bin; | |
| - name: Install helm | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| # install helm | |
| curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | |
| chmod 700 get_helm.sh | |
| ./get_helm.sh | |
| - name: Run K8s & Helm integration tests | |
| if: steps.changes.outputs.stack == 'true' | |
| timeout-minutes: 60 | |
| env: | |
| PYTEST_MODULES: "${{ matrix.pytest-modules }}" | |
| GITHUB_CI: true | |
| AZURE_BLOB_STORAGE_KEY: "${{ secrets.AZURE_BLOB_STORAGE_KEY }}" | |
| shell: bash | |
| run: | | |
| K3D_VERSION=v5.6.3 | |
| DEVSPACE_VERSION=v6.3.12 | |
| # install k3d | |
| wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64 | |
| mv k3d-linux-amd64 k3d | |
| chmod +x k3d | |
| export PATH=`pwd`:$PATH | |
| k3d version | |
| curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace | |
| chmod +x devspace | |
| devspace version | |
| tox -e stack.test.integration.k8s | |
| tox -e syft.build.helm | |
| tox -e syft.package.helm | |
| # tox -e syft.test.helm | |
| - name: Get current timestamp | |
| id: date | |
| if: failure() | |
| shell: bash | |
| run: echo "date=$(date +%s)" >> $GITHUB_OUTPUT | |
| - name: Collect logs from k3d | |
| if: steps.changes.outputs.stack == 'true' && failure() | |
| shell: bash | |
| run: | | |
| mkdir -p ./k8s-logs | |
| kubectl describe all -A --context k3d-test-gateway-1 --namespace syft > ./k8s-logs/test-gateway-1-desc-${{ steps.date.outputs.date }}.txt | |
| kubectl describe all -A --context k3d-test-domain-1 --namespace syft > ./k8s-logs/test-domain-1-desc-${{ steps.date.outputs.date }}.txt | |
| kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-test-gateway-1 --namespace syft > ./k8s-logs/test-gateway-1-logs-${{ steps.date.outputs.date }}.txt | |
| kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-test-domain-1 --namespace syft > ./k8s-logs/test-domain-1-logs-${{ steps.date.outputs.date }}.txt | |
| ls -la ./k8s-logs | |
| - name: Upload logs to GitHub | |
| uses: actions/upload-artifact@master | |
| if: steps.changes.outputs.stack == 'true' && failure() | |
| with: | |
| name: k8s-logs-integration-${{ matrix.os }}-${{ steps.date.outputs.date }} | |
| path: ./k8s-logs/ | |
| - name: Cleanup k3d | |
| if: steps.changes.outputs.stack == 'true' && failure() | |
| shell: bash | |
| run: | | |
| export PATH=`pwd`:$PATH | |
| k3d cluster delete test-gateway-1 || true | |
| k3d cluster delete test-domain-1 || true | |
| k3d registry delete k3d-registry.localhost || true | |
| pr-tests-notebook-k8s: | |
| strategy: | |
| max-parallel: 99 | |
| matrix: | |
| os: [ubuntu-latest] | |
| python-version: ["3.12"] | |
| fail-fast: false | |
| runs-on: ${{matrix.os}} | |
| steps: | |
| - name: Permission to home directory | |
| run: | | |
| sudo chown -R $USER:$USER $HOME | |
| - uses: actions/checkout@v4 | |
| - name: Check for file changes | |
| uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| base: ${{ github.ref }} | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| if: steps.changes.outputs.stack == 'true' | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Add K3d Registry | |
| run: | | |
| sudo python ./scripts/patch_hosts.py --add-k3d-registry | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: true | |
| large-packages: false | |
| # free 10GB of space | |
| - name: Remove unnecessary files | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| docker image prune --all --force | |
| docker builder prune --all --force | |
| docker system prune --all --force | |
| - name: Upgrade pip | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| pip install --upgrade pip uv==0.1.35 | |
| uv --version | |
| - name: Get pip cache dir | |
| if: steps.changes.outputs.stack == 'true' | |
| id: pip-cache | |
| shell: bash | |
| run: | | |
| echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT | |
| - name: pip cache | |
| uses: actions/cache@v4 | |
| if: steps.changes.outputs.stack == 'true' | |
| with: | |
| path: ${{ steps.pip-cache.outputs.dir }} | |
| key: ${{ runner.os }}-uv-py${{ matrix.python-version }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-py${{ matrix.python-version }} | |
| - name: Install tox | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| pip install --upgrade tox tox-uv==1.5.1 | |
| - name: Install kubectl | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| # cleanup apt version | |
| sudo apt remove kubectl || true | |
| # install kubectl 1.27 | |
| curl -LO https://dl.k8s.io/release/v1.27.2/bin/linux/amd64/kubectl | |
| chmod +x kubectl | |
| sudo install kubectl /usr/local/bin; | |
| - name: Install helm | |
| if: steps.changes.outputs.stack == 'true' | |
| run: | | |
| # install helm | |
| curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | |
| chmod 700 get_helm.sh | |
| ./get_helm.sh | |
| - name: Run Notebooks Tests | |
| if: steps.changes.outputs.stack == 'true' | |
| timeout-minutes: 60 | |
| env: | |
| GITHUB_CI: true | |
| shell: bash | |
| run: | | |
| K3D_VERSION=v5.6.3 | |
| DEVSPACE_VERSION=v6.3.12 | |
| # install k3d | |
| wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64 | |
| mv k3d-linux-amd64 k3d | |
| chmod +x k3d | |
| export PATH=`pwd`:$PATH | |
| k3d version | |
| curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace | |
| chmod +x devspace | |
| devspace version | |
| tox -e stack.test.notebook.k8s | |
| - name: Get current timestamp | |
| id: date | |
| if: failure() | |
| shell: bash | |
| run: echo "date=$(date +%s)" >> $GITHUB_OUTPUT | |
| - name: Collect logs from k3d | |
| if: steps.changes.outputs.stack == 'true' && failure() | |
| shell: bash | |
| run: | | |
| mkdir -p ./k8s-logs | |
| kubectl describe all -A --context k3d-test-gateway-1 --namespace syft > ./k8s-logs/test-gateway-1-desc-${{ steps.date.outputs.date }}.txt | |
| kubectl describe all -A --context k3d-test-domain-1 --namespace syft > ./k8s-logs/test-domain-1-desc-${{ steps.date.outputs.date }}.txt | |
| kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-test-gateway-1 --namespace syft > ./k8s-logs/test-gateway-1-logs-${{ steps.date.outputs.date }}.txt | |
| kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-test-domain-1 --namespace syft > ./k8s-logs/test-domain-1-logs-${{ steps.date.outputs.date }}.txt | |
| ls -la ./k8s-logs | |
| - name: Upload logs to GitHub | |
| uses: actions/upload-artifact@master | |
| if: steps.changes.outputs.stack == 'true' && failure() | |
| with: | |
| name: k8s-logs-notebook-${{ matrix.os }}-${{ steps.date.outputs.date }} | |
| path: ./k8s-logs/ | |
| - name: Cleanup k3d | |
| if: steps.changes.outputs.stack == 'true' && failure() | |
| shell: bash | |
| run: | | |
| export PATH=`pwd`:$PATH | |
| k3d cluster delete test-gateway-1 || true | |
| k3d cluster delete test-domain-1 || true | |
| k3d registry delete k3d-registry.localhost || true |