[DEEP-112] force cleanup is RAM is filled more than 90% #8612
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| permissions: | |
| id-token: write | |
| contents: read | |
| checks: write | |
| pull-requests: write | |
| on: | |
| push: | |
| branches: | |
| - 'main' | |
| - 'release/**' | |
| pull_request: | |
| types: | |
| - opened | |
| - reopened | |
| - synchronize | |
| - labeled | |
| - ready_for_review | |
| concurrency: | |
| cancel-in-progress: true | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' || (! github.event.pull_request.draft) | |
| outputs: | |
| matrix: ${{ steps.setup.outputs.matrix }} | |
| steps: | |
| - name: Setup tests matrix | |
| id: setup | |
| shell: python | |
| run: | | |
| import os | |
| import json | |
| matrix = { | |
| "os": [ | |
| "ubuntu-latest", | |
| "windows-latest", | |
| ], | |
| "python-version": [ | |
| "3.11", | |
| ], | |
| "storage": [ | |
| "--hub-cloud --local", | |
| "--s3 --gcs --azure --gdrive --memory-skip" | |
| ] | |
| } | |
| if "${{ github.event_name }}" == "pull_request": | |
| if "${{ contains(github.event.pull_request.labels.*.name, 'quick-tests') }}" == "true": | |
| matrix = { | |
| "os": [ | |
| "ubuntu-latest", | |
| ], | |
| "python-version": [ | |
| "3.11", | |
| ], | |
| "storage": [ | |
| "local", | |
| ] | |
| } | |
| if "${{ contains(github.event.pull_request.labels.*.name, 'macos-tests') }}" == "true": | |
| matrix["os"].append("macos-latest") | |
| if "${{ contains(github.event.pull_request.labels.*.name, 'full-tests') }}" == "true": | |
| matrix["os"] = [ | |
| "macos-latest", | |
| "ubuntu-latest", | |
| "windows-latest", | |
| ] | |
| matrix["python-version"] = [ | |
| "3.11", | |
| "3.10", | |
| "3.9", | |
| "3.8", | |
| ] | |
| matrix["storage"] = [ | |
| "--hub-cloud --local", | |
| "--s3 --gcs --azure --gdrive --memory-skip" | |
| ] | |
| elif "${{ github.event_name }}" == "push": | |
| if "${{ github.ref_name == 'main' || startsWith( github.ref_name, 'release/' ) }}" == "true": | |
| matrix["os"] = [ | |
| "macos-latest", | |
| "ubuntu-latest", | |
| "windows-latest", | |
| ] | |
| matrix["python-version"] = [ | |
| "3.11", | |
| "3.10", | |
| "3.9", | |
| "3.8", | |
| ] | |
| else: | |
| raise Exception("Unexpected branch") | |
| else: | |
| raise Exception("Unexpected event") | |
| matrix["exclude"] = [ | |
| { | |
| "os": "macos-latest", | |
| "python-version": "3.8", | |
| }, | |
| { | |
| "os": "macos-latest", | |
| "python-version": "3.9", | |
| }, | |
| { | |
| "os": "macos-latest", | |
| "python-version": "3.11", | |
| }, | |
| { | |
| "os": "windows-latest", | |
| "python-version": "3.11", | |
| }, | |
| ] | |
| with open(os.environ.get("GITHUB_OUTPUT"), "a") as file: | |
| file.write("matrix=%s\n" % json.dumps(matrix)) | |
| test: | |
| name: Test | |
| needs: setup | |
| uses: activeloopai/shared-github-actions/.github/workflows/full_test.yml@main | |
| if: github.repository == 'activeloopai/deeplake' | |
| with: | |
| repo: ${{ github.repository }} | |
| ref: ${{ github.sha }} | |
| testMatrix: ${{ needs.setup.outputs.matrix }} | |
| secrets: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| aws_role_arn: ${{ secrets.AWS_ROLE_ARN }} | |
| gcp_sa_credentials_json: ${{ secrets.GCP_SA_CREDENTIALS_JSON }} | |
| azure_creds_json: ${{ secrets.AZURE_CREDS_JSON }} | |
| hub_username: ${{ secrets.ACTIVELOOP_HUB_USERNAME }} | |
| hub_token: ${{ secrets.ACTIVELOOP_HUB_TOKEN }} | |
| kaggle_username: ${{ secrets.KAGGLE_USERNAME }} | |
| kaggle_key: ${{ secrets.KAGGLE_KEY }} | |
| oauth_client_id: ${{ secrets.GDRIVE_CLIENT_ID }} | |
| oauth_client_secret: ${{ secrets.GDRIVE_CLIENT_SECRET }} | |
| oauth_refresh_token: ${{ secrets.GDRIVE_REFRESH_TOKEN }} | |
| sonar_token: ${{ secrets.SONAR_TOKEN }} |