From 3cb51a63ae5e7b9b8f5a83767377d9848b3ad766 Mon Sep 17 00:00:00 2001 From: Ilya Gurov Date: Thu, 28 Jan 2021 12:42:39 +0300 Subject: [PATCH] test: use parallel workflows to run Django tests (#569) Move emulator and non-emulator tests into separate workflows, split emulator tests across multiple github workflows, disable several tests that are incompatible with the Cloud Spanner API. --- .../django_tests_against_emulator0.yml | 32 +++++ .../django_tests_against_emulator1.yml | 32 +++++ .../django_tests_against_emulator10.yml | 32 +++++ .../django_tests_against_emulator11.yml | 32 +++++ .../django_tests_against_emulator12.yml | 32 +++++ .../django_tests_against_emulator13.yml | 32 +++++ .../django_tests_against_emulator14.yml | 32 +++++ .../django_tests_against_emulator15.yml | 32 +++++ .../django_tests_against_emulator16.yml | 32 +++++ .../django_tests_against_emulator17.yml | 32 +++++ .../django_tests_against_emulator18.yml | 32 +++++ .../django_tests_against_emulator2.yml | 32 +++++ .../django_tests_against_emulator3.yml | 32 +++++ .../django_tests_against_emulator4.yml | 32 +++++ .../django_tests_against_emulator5.yml | 32 +++++ .../django_tests_against_emulator6.yml | 32 +++++ .../django_tests_against_emulator7.yml | 32 +++++ .../django_tests_against_emulator8.yml | 32 +++++ .../django_tests_against_emulator9.yml | 32 +++++ .kokoro/build.sh | 32 ++--- create_test_instance.py | 33 +++++ django_spanner/base.py | 10 +- django_spanner/compiler.py | 6 +- django_spanner/features.py | 119 +++++++++++++-- django_test_apps.txt | 135 +----------------- django_test_suite.sh | 53 ++++--- setup.py | 2 +- 27 files changed, 807 insertions(+), 191 deletions(-) create mode 100644 .github/workflows/django_tests_against_emulator0.yml create mode 100644 .github/workflows/django_tests_against_emulator1.yml create mode 100644 .github/workflows/django_tests_against_emulator10.yml create mode 100644 .github/workflows/django_tests_against_emulator11.yml create mode 100644 .github/workflows/django_tests_against_emulator12.yml create mode 100644 .github/workflows/django_tests_against_emulator13.yml create mode 100644 .github/workflows/django_tests_against_emulator14.yml create mode 100644 .github/workflows/django_tests_against_emulator15.yml create mode 100644 .github/workflows/django_tests_against_emulator16.yml create mode 100644 .github/workflows/django_tests_against_emulator17.yml create mode 100644 .github/workflows/django_tests_against_emulator18.yml create mode 100644 .github/workflows/django_tests_against_emulator2.yml create mode 100644 .github/workflows/django_tests_against_emulator3.yml create mode 100644 .github/workflows/django_tests_against_emulator4.yml create mode 100644 .github/workflows/django_tests_against_emulator5.yml create mode 100644 .github/workflows/django_tests_against_emulator6.yml create mode 100644 .github/workflows/django_tests_against_emulator7.yml create mode 100644 .github/workflows/django_tests_against_emulator8.yml create mode 100644 .github/workflows/django_tests_against_emulator9.yml create mode 100644 create_test_instance.py diff --git a/.github/workflows/django_tests_against_emulator0.yml b/.github/workflows/django_tests_against_emulator0.yml new file mode 100644 index 0000000000..6e02edd369 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator0.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests0 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: model_fields model_forms diff --git a/.github/workflows/django_tests_against_emulator1.yml b/.github/workflows/django_tests_against_emulator1.yml new file mode 100644 index 0000000000..a8689d71c5 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator1.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests1 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: mutually_referential nested_foreign_keys null_fk null_fk_ordering null_queries one_to_one ordering order_with_respect_to or_lookups diff --git a/.github/workflows/django_tests_against_emulator10.yml b/.github/workflows/django_tests_against_emulator10.yml new file mode 100644 index 0000000000..f2dae26f52 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator10.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests10 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: select_related select_related_onetoone select_related_regress queryset_pickle diff --git a/.github/workflows/django_tests_against_emulator11.yml b/.github/workflows/django_tests_against_emulator11.yml new file mode 100644 index 0000000000..6c6cac8b9a --- /dev/null +++ b/.github/workflows/django_tests_against_emulator11.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests11 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: transaction_hooks transactions unmanaged_models update update_only_fields validation admin_changelist admin_docs view_tests many_to_many many_to_one many_to_one_null diff --git a/.github/workflows/django_tests_against_emulator12.yml b/.github/workflows/django_tests_against_emulator12.yml new file mode 100644 index 0000000000..26d4151b76 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator12.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests12 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: aggregation_regress annotations auth_tests backends basic bulk_create cache choices constraints diff --git a/.github/workflows/django_tests_against_emulator13.yml b/.github/workflows/django_tests_against_emulator13.yml new file mode 100644 index 0000000000..81c769aa4f --- /dev/null +++ b/.github/workflows/django_tests_against_emulator13.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests13 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: delete delete_regress distinct_on_fields empty expressions expressions_window field_defaults file_storage file_uploads filtered_relation diff --git a/.github/workflows/django_tests_against_emulator14.yml b/.github/workflows/django_tests_against_emulator14.yml new file mode 100644 index 0000000000..74b91c3f06 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator14.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests14 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: foreign_object forms_tests from_db_value generic_inline_admin generic_relations generic_relations_regress diff --git a/.github/workflows/django_tests_against_emulator15.yml b/.github/workflows/django_tests_against_emulator15.yml new file mode 100644 index 0000000000..b1bb1b921e --- /dev/null +++ b/.github/workflows/django_tests_against_emulator15.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests15 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: known_related_objects lookup m2m_and_m2o m2m_intermediary m2m_multiple m2m_recursive m2m_regress m2m_signals m2m_through m2m_through_regress m2o_recursive managers_regress diff --git a/.github/workflows/django_tests_against_emulator16.yml b/.github/workflows/django_tests_against_emulator16.yml new file mode 100644 index 0000000000..1868e89efa --- /dev/null +++ b/.github/workflows/django_tests_against_emulator16.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests16 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: model_formsets model_formsets_regress model_indexes diff --git a/.github/workflows/django_tests_against_emulator17.yml b/.github/workflows/django_tests_against_emulator17.yml new file mode 100644 index 0000000000..f310238593 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator17.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests17 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: servers sessions_tests signals diff --git a/.github/workflows/django_tests_against_emulator18.yml b/.github/workflows/django_tests_against_emulator18.yml new file mode 100644 index 0000000000..5f8a0b9dde --- /dev/null +++ b/.github/workflows/django_tests_against_emulator18.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests18 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: redirects_tests reserved_names reverse_lookup save_delete_hooks schema raw_query contenttypes_tests diff --git a/.github/workflows/django_tests_against_emulator2.yml b/.github/workflows/django_tests_against_emulator2.yml new file mode 100644 index 0000000000..8703d275ed --- /dev/null +++ b/.github/workflows/django_tests_against_emulator2.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests2 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: migration_test_data_persistence max_lengths migrate_signals migrations select_for_update queries diff --git a/.github/workflows/django_tests_against_emulator3.yml b/.github/workflows/django_tests_against_emulator3.yml new file mode 100644 index 0000000000..a7cd8702d0 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator3.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests3 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: sitemaps_tests sites_framework sites_tests string_lookup syndication_tests test_client test_client_regress test_runner test_utils timezones diff --git a/.github/workflows/django_tests_against_emulator4.yml b/.github/workflows/django_tests_against_emulator4.yml new file mode 100644 index 0000000000..c0a04f17e5 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator4.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests4 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: admin_inlines admin_ordering aggregation diff --git a/.github/workflows/django_tests_against_emulator5.yml b/.github/workflows/django_tests_against_emulator5.yml new file mode 100644 index 0000000000..e5ec591a84 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator5.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests5 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: custom_columns custom_managers custom_methods custom_pk datatypes dates datetimes db_functions defer defer_regress diff --git a/.github/workflows/django_tests_against_emulator6.yml b/.github/workflows/django_tests_against_emulator6.yml new file mode 100644 index 0000000000..1eff8a53c2 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator6.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests6 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: fixtures fixtures_model_package fixtures_regress flatpages_tests force_insert_update diff --git a/.github/workflows/django_tests_against_emulator7.yml b/.github/workflows/django_tests_against_emulator7.yml new file mode 100644 index 0000000000..e4ceed9d0d --- /dev/null +++ b/.github/workflows/django_tests_against_emulator7.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests7 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: generic_views get_earliest_or_latest get_object_or_404 get_or_create i18n indexes inline_formsets inspectdb introspection invalid_models_tests diff --git a/.github/workflows/django_tests_against_emulator8.yml b/.github/workflows/django_tests_against_emulator8.yml new file mode 100644 index 0000000000..4a358f9654 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator8.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests8 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: model_inheritance model_inheritance_regress model_options model_package multiple_database diff --git a/.github/workflows/django_tests_against_emulator9.yml b/.github/workflows/django_tests_against_emulator9.yml new file mode 100644 index 0000000000..a00006f932 --- /dev/null +++ b/.github/workflows/django_tests_against_emulator9.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - master + pull_request: +name: django-tests9 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: prefetch_related proxy_model_inheritance diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 8d7113079a..09323d7da1 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -28,6 +28,7 @@ export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json # Setup project id. export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") +export GOOGLE_CLOUD_PROJECT=$(cat "${KOKORO_GFILE_DIR}/project-id.json") # Remove old nox python3.6 -m pip uninstall --yes --quiet nox-automation @@ -54,6 +55,7 @@ export RUNNING_SPANNER_BACKEND_TESTS=1 export USE_SPANNER_EMULATOR=0 pip3 install . +pip3 install -e 'git+https://github.com/q-logic/python-spanner-django.git@dj_tests_against_emulator#egg=django-google-spanner' # Create a unique DJANGO_TESTS_DIR per worker to avoid # any clashes with configured tests by other workers. export DJANGO_TESTS_DIR="django_tests_$DJANGO_WORKER_INDEX" @@ -64,28 +66,18 @@ sudo apt-get update apt-get install -y libffi-dev libjpeg-dev zlib1g-dev libmemcached-dev cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../ -if [[ $USE_SPANNER_EMULATOR != 1 ]] +# Not using the emulator! +# Hardcode the max number of workers since Spanner has a very low +# QPS for administrative RPCs of 5QPS (averaged every 100 seconds) +if [[ $KOKORO_JOB_NAME == *"continuous"* ]] then - # Not using the emulator! - # Hardcode the max number of workers since Spanner has a very low - # QPS for administrative RPCs of 5QPS (averaged every 100 seconds) - if [[ $KOKORO_JOB_NAME == *"continuous"* ]] - then - # Disable continuous build as it creates too many Spanner instances - # ("Quota exceeded for quota metric 'Instance create requests' and - # limit 'Instance create requests per minute' of service - # 'spanner.googleapis.com'). - export DJANGO_WORKER_COUNT=0 - else - export DJANGO_WORKER_COUNT=6 - fi + # Disable continuous build as it creates too many Spanner instances + # ("Quota exceeded for quota metric 'Instance create requests' and + # limit 'Instance create requests per minute' of service + # 'spanner.googleapis.com'). + export DJANGO_WORKER_COUNT=0 else - export DJANGO_WORKER_COUNT=$(ls .kokoro/presubmit/worker* | wc -l) - # Install and start the Spanner emulator - VERSION=0.7.3 - wget https://storage.googleapis.com/cloud-spanner-emulator/releases/${VERSION}/cloud-spanner-emulator_linux_amd64-${VERSION}.tar.gz 2&>/dev/null - tar zxvf cloud-spanner-emulator_linux_amd64-${VERSION}.tar.gz 2&>/dev/null - chmod +x emulator_main + export DJANGO_WORKER_COUNT=4 fi ./bin/parallelize_tests_linux diff --git a/create_test_instance.py b/create_test_instance.py new file mode 100644 index 0000000000..38a7153364 --- /dev/null +++ b/create_test_instance.py @@ -0,0 +1,33 @@ +# Copyright 2016 Google LLC All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud.spanner_v1 import Client + +if os.getenv( + "SPANNER_EMULATOR_HOST" +): # otherwise instance will be created by parallelize_tests + project = os.getenv( + "GOOGLE_CLOUD_PROJECT", + os.getenv("PROJECT_ID", "emulator-test-project"), + ) + + client = Client(project=project) + + config = f"{client.project_name}/instanceConfigs/regional-us-central1" + + instance = client.instance("google-cloud-django-backend-tests", config) + created_op = instance.create() + created_op.result(30) # block until completion diff --git a/django_spanner/base.py b/django_spanner/base.py index 35743d0385..0ede662571 100644 --- a/django_spanner/base.py +++ b/django_spanner/base.py @@ -4,7 +4,9 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd -import google.cloud.spanner_v1 as spanner +import os + +from google.cloud import spanner from django.db.backends.base.base import BaseDatabaseWrapper from google.cloud import spanner_dbapi @@ -111,7 +113,9 @@ def instance(self): :rtype: :class:`~google.cloud.spanner_v1.instance.Instance` :returns: A new instance owned by the existing Spanner Client. """ - return spanner.Client().instance(self.settings_dict["INSTANCE"]) + return spanner.Client( + project=os.environ["GOOGLE_CLOUD_PROJECT"] + ).instance(self.settings_dict["INSTANCE"]) @property def _nodb_connection(self): @@ -127,7 +131,7 @@ def get_connection_params(self): in Django Spanner format. """ return { - "project": self.settings_dict["PROJECT"], + "project": os.environ["GOOGLE_CLOUD_PROJECT"], "instance_id": self.settings_dict["INSTANCE"], "database_id": self.settings_dict["NAME"], "user_agent": "django_spanner/2.2.0a1", diff --git a/django_spanner/compiler.py b/django_spanner/compiler.py index 4cd7dfb877..4b584e8e5b 100644 --- a/django_spanner/compiler.py +++ b/django_spanner/compiler.py @@ -13,7 +13,6 @@ SQLUpdateCompiler as BaseSQLUpdateCompiler, ) from django.db.utils import DatabaseError -from django_spanner.utils import add_dummy_where class SQLCompiler(BaseSQLCompiler): @@ -108,25 +107,28 @@ def get_combinator_sql(self, combinator, all): for part in args_parts: params.extend(part) - result = add_dummy_where(result) return result, params class SQLInsertCompiler(BaseSQLInsertCompiler, SQLCompiler): """A wrapper class for compatibility with Django specifications.""" + pass class SQLDeleteCompiler(BaseSQLDeleteCompiler, SQLCompiler): """A wrapper class for compatibility with Django specifications.""" + pass class SQLUpdateCompiler(BaseSQLUpdateCompiler, SQLCompiler): """A wrapper class for compatibility with Django specifications.""" + pass class SQLAggregateCompiler(BaseSQLAggregateCompiler, SQLCompiler): """A wrapper class for compatibility with Django specifications.""" + pass diff --git a/django_spanner/features.py b/django_spanner/features.py index 98e3bc3daf..b8daca44bb 100644 --- a/django_spanner/features.py +++ b/django_spanner/features.py @@ -11,7 +11,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): - """An extension to Django database feature descriptor.""" can_introspect_big_integer_field = False can_introspect_duration_field = False can_introspect_foreign_keys = False @@ -22,7 +21,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): # Spanner uses REGEXP_CONTAINS which is case-sensitive. has_case_insensitive_like = False # https://cloud.google.com/spanner/quotas#query_limits - max_query_params = 950 + max_query_params = 900 supports_foreign_keys = False supports_ignore_conflicts = False supports_partial_indexes = False @@ -44,7 +43,12 @@ class DatabaseFeatures(BaseDatabaseFeatures): "basic.tests.SelectOnSaveTests.test_select_on_save_lying_update", # django_spanner monkey patches AutoField to have a default value. "basic.tests.ModelTest.test_hash", + "custom_managers.tests.CustomManagerTests.test_slow_removal_through_specified_fk_related_manager", + "custom_managers.tests.CustomManagerTests.test_slow_removal_through_default_fk_related_manager", "generic_relations.test_forms.GenericInlineFormsetTests.test_options", + "generic_relations.tests.GenericRelationsTests.test_add_bulk_false", + "generic_relations.tests.GenericRelationsTests.test_generic_update_or_create_when_updated", + "generic_relations.tests.GenericRelationsTests.test_update_or_create_defaults", "generic_relations.tests.GenericRelationsTests.test_unsaved_instance_on_generic_foreign_key", "generic_relations_regress.tests.GenericRelationTests.test_target_model_is_unsaved", "m2m_through_regress.tests.ToFieldThroughTests.test_m2m_relations_unusable_on_null_pk_obj", @@ -57,17 +61,29 @@ class DatabaseFeatures(BaseDatabaseFeatures): "model_inheritance.tests.ModelInheritanceTests.test_create_child_no_update", "model_regress.tests.ModelTests.test_get_next_prev_by_field_unsaved", "one_to_one.tests.OneToOneTests.test_get_reverse_on_unsaved_object", + "one_to_one.tests.OneToOneTests.test_o2o_primary_key_delete", "one_to_one.tests.OneToOneTests.test_set_reverse_on_unsaved_object", "one_to_one.tests.OneToOneTests.test_unsaved_object", "queries.test_bulk_update.BulkUpdateNoteTests.test_unsaved_models", + "expressions_case.tests.CaseExpressionTests.test_update_decimal", "serializers.test_json.JsonSerializerTestCase.test_pkless_serialized_strings", "serializers.test_json.JsonSerializerTestCase.test_serialize_with_null_pk", "serializers.test_xml.XmlSerializerTestCase.test_pkless_serialized_strings", "serializers.test_xml.XmlSerializerTestCase.test_serialize_with_null_pk", "serializers.test_yaml.YamlSerializerTestCase.test_pkless_serialized_strings", "serializers.test_yaml.YamlSerializerTestCase.test_serialize_with_null_pk", + "serializers.test_data.SerializerDataTests.test_yaml_serializer", + "serializers.test_data.SerializerDataTests.test_xml_serializer", + "serializers.test_data.SerializerDataTests.test_python_serializer", + "serializers.test_data.SerializerDataTests.test_json_serializer", "timezones.tests.LegacyDatabaseTests.test_cursor_execute_accepts_naive_datetime", "timezones.tests.NewDatabaseTests.test_cursor_execute_accepts_naive_datetime", + "timezones.tests.AdminTests.test_change_editable", + "timezones.tests.AdminTests.test_change_editable_in_other_timezone", + "timezones.tests.AdminTests.test_change_readonly", + "timezones.tests.AdminTests.test_change_readonly_in_other_timezone", + "timezones.tests.AdminTests.test_changelist", + "timezones.tests.AdminTests.test_changelist_in_other_timezone", "validation.test_custom_messages.CustomMessagesTests.test_custom_null_message", "validation.test_custom_messages.CustomMessagesTests.test_custom_simple_validator_message", "validation.test_unique.PerformUniqueChecksTest.test_primary_key_unique_check_not_performed_when_adding_and_pk_not_specified", # noqa @@ -89,9 +105,20 @@ class DatabaseFeatures(BaseDatabaseFeatures): "admin_filters.tests.ListFiltersTests.test_relatedfieldlistfilter_manytomany", "admin_filters.tests.ListFiltersTests.test_simplelistfilter", "admin_inlines.tests.TestInline.test_inline_hidden_field_no_column", + "proxy_models.tests.ProxyModelAdminTests.test_delete_str_in_model_admin", "admin_utils.test_logentry.LogEntryTests.test_logentry_change_message", "admin_utils.test_logentry.LogEntryTests.test_logentry_change_message_localized_datetime_input", "admin_utils.test_logentry.LogEntryTests.test_proxy_model_content_type_is_used_for_log_entries", + "admin_utils.test_logentry.LogEntryTests.test_action_flag_choices", + "admin_utils.test_logentry.LogEntryTests.test_log_action", + "admin_utils.test_logentry.LogEntryTests.test_logentry_change_message_formsets", + "admin_utils.test_logentry.LogEntryTests.test_logentry_change_message_not_json", + "admin_utils.test_logentry.LogEntryTests.test_logentry_get_admin_url", + "admin_utils.test_logentry.LogEntryTests.test_logentry_get_edited_object", + "admin_utils.test_logentry.LogEntryTests.test_logentry_repr", + "admin_utils.test_logentry.LogEntryTests.test_logentry_save", + "admin_utils.test_logentry.LogEntryTests.test_logentry_unicode", + "admin_utils.test_logentry.LogEntryTests.test_recentactions_without_content_type", "admin_views.tests.AdminViewPermissionsTest.test_history_view", "aggregation.test_filter_argument.FilteredAggregateTests.test_plain_annotate", "aggregation.tests.AggregateTestCase.test_annotate_basic", @@ -100,6 +127,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): "aggregation_regress.tests.AggregationTests.test_more_more", "aggregation_regress.tests.AggregationTests.test_more_more_more", "aggregation_regress.tests.AggregationTests.test_ticket_11293", + "defer_regress.tests.DeferRegressionTest.test_ticket_12163", "defer_regress.tests.DeferRegressionTest.test_ticket_23270", "distinct_on_fields.tests.DistinctOnTests.test_basic_distinct_on", "extra_regress.tests.ExtraRegressTests.test_regression_7314_7372", @@ -225,6 +253,10 @@ class DatabaseFeatures(BaseDatabaseFeatures): "cache.tests.CreateCacheTableForDBCacheTests", "cache.tests.DBCacheTests", "cache.tests.DBCacheWithTimeZoneTests", + "delete.tests.DeletionTests.test_queryset_delete_returns_num_rows", + "delete.tests.DeletionTests.test_model_delete_returns_num_rows", + "delete.tests.DeletionTests.test_deletion_order", + "delete.tests.FastDeleteTests.test_fast_delete_empty_no_update_can_self_select", # Tests that require transactions. "transaction_hooks.tests.TestConnectionOnCommit.test_does_not_execute_if_transaction_rolled_back", "transaction_hooks.tests.TestConnectionOnCommit.test_hooks_cleared_after_rollback", @@ -324,20 +356,25 @@ class DatabaseFeatures(BaseDatabaseFeatures): "model_inheritance_regress.tests.ModelInheritanceTest.test_issue_6755", # Probably due to django-spanner setting a default on AutoField: # https://github.com/googleapis/python-spanner-django/issues/424 + "model_forms.tests.ModelFormBasicTests.test_runtime_choicefield_populated", + "model_forms.tests.ModelFormBasicTests.test_multi_fields", + "model_forms.tests.ModelFormBasicTests.test_m2m_initial_callable", + "model_forms.tests.ModelFormBasicTests.test_initial_values", + "model_forms.tests.OtherModelFormTests.test_prefetch_related_queryset", "model_formsets.tests.ModelFormsetTest.test_prevent_change_outer_model_and_create_invalid_data", "model_formsets_regress.tests.FormfieldShouldDeleteFormTests.test_no_delete", "model_formsets_regress.tests.FormsetTests.test_extraneous_query_is_not_run", + # os.chmod() doesn't work on Kokoro? + "file_uploads.tests.DirectoryCreationTests.test_readonly_root", + # Tests that sometimes fail on Kokoro for unknown reasons. + "contenttypes_tests.test_models.ContentTypesTests.test_cache_not_shared_between_managers", + "migration_test_data_persistence.tests.MigrationDataNormalPersistenceTestCase.test_persistence", + "servers.test_liveserverthread.LiveServerThreadTest.test_closes_connections", + "servers.tests.LiveServerDatabase.test_fixtures_loaded", + "view_tests.tests.test_csrf.CsrfViewTests.test_no_cookies", + "view_tests.tests.test_csrf.CsrfViewTests.test_no_referer", + "view_tests.tests.test_i18n.SetLanguageTests.test_lang_from_translated_i18n_pattern", ) - # Kokoro-specific skips. - if os.environ.get("KOKORO_JOB_NAME"): - skip_tests += ( - # os.chmod() doesn't work on Kokoro? - "file_uploads.tests.DirectoryCreationTests.test_readonly_root", - # Tests that sometimes fail on Kokoro for unknown reasons. - "contenttypes_tests.test_models.ContentTypesTests.test_cache_not_shared_between_managers", - "migration_test_data_persistence.tests.MigrationDataNormalPersistenceTestCase.test_persistence", - "servers.test_liveserverthread.LiveServerThreadTest.test_closes_connections", - ) if os.environ.get("SPANNER_EMULATOR_HOST", None): # Some code isn't yet supported by the Spanner emulator. @@ -556,6 +593,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): "aggregation_regress.tests.SelfReferentialFKTests.test_ticket_24748", # noqa "annotations.tests.NonAggregateAnnotationTestCase.test_custom_functions", # noqa "annotations.tests.NonAggregateAnnotationTestCase.test_custom_functions_can_ref_other_functions", # noqa + "annotations.tests.NonAggregateAnnotationTestCase.test_filter_decimal_annotation", # noqa "auth_tests.test_auth_backends.AllowAllUsersModelBackendTest.test_authenticate", # noqa "auth_tests.test_auth_backends.AllowAllUsersModelBackendTest.test_get_user", # noqa "auth_tests.test_auth_backends.AuthenticateTests.test_skips_backends_without_arguments", # noqa @@ -733,6 +771,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): "auth_tests.test_migrations.ProxyModelWithDifferentAppLabelTests.test_user_keeps_same_permissions_after_migrating_backward", # noqa "auth_tests.test_migrations.ProxyModelWithSameAppLabelTests.test_user_keeps_same_permissions_after_migrating_backward", # noqa "auth_tests.test_migrations.ProxyModelWithSameAppLabelTests.test_user_still_has_proxy_model_permissions", # noqa + "proxy_models.tests.ProxyModelAdminTests.test_delete_str_in_model_admin", # noqa "auth_tests.test_mixins.AccessMixinTests.test_access_mixin_permission_denied_response", # noqa "auth_tests.test_mixins.AccessMixinTests.test_stacked_mixins_missing_permission", # noqa "auth_tests.test_mixins.AccessMixinTests.test_stacked_mixins_not_logged_in", # noqa @@ -923,6 +962,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): "bulk_create.tests.BulkCreateTests.test_bulk_insert_nullable_fields", # noqa "custom_pk.tests.CustomPKTests.test_required_pk", # noqa "custom_pk.tests.CustomPKTests.test_unique_pk", # noqa + "custom_lookups.tests.SubqueryTransformTests.test_subquery_usage", # noqa "datatypes.tests.DataTypesTestCase.test_boolean_type", # noqa "datatypes.tests.DataTypesTestCase.test_date_type", # noqa "datatypes.tests.DataTypesTestCase.test_textfields_str", # noqa @@ -1051,6 +1091,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): "db_functions.text.test_upper.UpperTests.test_transform", # noqa "defer_regress.tests.DeferAnnotateSelectRelatedTest.test_defer_annotate_select_related", # noqa "delete_regress.tests.DeleteCascadeTransactionTests.test_inheritance", # noqa + "delete_regress.tests.DeleteLockingTest.test_concurrent_delete", # noqa "expressions.test_queryset_values.ValuesExpressionsTests.test_chained_values_with_expression", # noqa "expressions.test_queryset_values.ValuesExpressionsTests.test_values_expression", # noqa "expressions.test_queryset_values.ValuesExpressionsTests.test_values_expression_group_by", # noqa @@ -1107,11 +1148,33 @@ class DatabaseFeatures(BaseDatabaseFeatures): "expressions.tests.ValueTests.test_update_TimeField_using_Value", # noqa "expressions.tests.ValueTests.test_update_UUIDField_using_Value", # noqa "fixtures.tests.FixtureLoadingTests.test_loaddata_error_message", # noqa + "fixtures.tests.FixtureLoadingTests.test_ambiguous_compressed_fixture", # noqa + "fixtures.tests.FixtureTransactionTests.test_format_discovery", # noqa "fixtures.tests.ForwardReferenceTests.test_forward_reference_fk", # noqa "fixtures.tests.ForwardReferenceTests.test_forward_reference_m2m", # noqa + "flatpages_tests.test_csrf.FlatpageCSRFTests.test_view_authenticated_flatpage", # noqa + "flatpages_tests.test_middleware.FlatpageMiddlewareTests.test_fallback_authenticated_flatpage", # noqa + "flatpages_tests.test_middleware.FlatpageMiddlewareTests.test_view_authenticated_flatpage", # noqa + "flatpages_tests.test_templatetags.FlatpageTemplateTagTests.test_get_flatpages_tag_for_user", # noqa + "flatpages_tests.test_templatetags.FlatpageTemplateTagTests.test_get_flatpages_with_prefix_for_user", # noqa + "flatpages_tests.test_views.FlatpageViewTests.test_view_authenticated_flatpage", # noqa + "generic_inline_admin.tests.GenericAdminViewTest.test_basic_add_GET", # noqa + "generic_inline_admin.tests.GenericAdminViewTest.test_basic_add_POST", # noqa + "generic_inline_admin.tests.GenericAdminViewTest.test_basic_edit_GET", # noqa + "generic_inline_admin.tests.GenericAdminViewTest.test_basic_edit_POST", # noqa + "generic_inline_admin.tests.GenericInlineAdminParametersTest.testMaxNumParam", # noqa + "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_get_extra", # noqa + "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_extra_param", # noqa + "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_get_max_num", # noqa + "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_get_min_num", # noqa + "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_min_num_param", # noqa + "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_no_param", # noqa + "generic_inline_admin.tests.GenericInlineAdminWithUniqueTogetherTest.test_add", # noqa + "generic_inline_admin.tests.GenericInlineAdminWithUniqueTogetherTest.test_delete", # noqa "get_or_create.tests.GetOrCreateTests.test_get_or_create_invalid_params", # noqa "get_or_create.tests.GetOrCreateTestsWithManualPKs.test_create_with_duplicate_primary_key", # noqa "get_or_create.tests.GetOrCreateTestsWithManualPKs.test_get_or_create_raises_IntegrityError_plus_traceback", # noqa + "i18n.tests.WatchForTranslationChangesTests.test_i18n_app_dirs", # noqa "introspection.tests.IntrospectionTests.test_get_constraints", # noqa "introspection.tests.IntrospectionTests.test_get_constraints_index_types", # noqa "introspection.tests.IntrospectionTests.test_get_constraints_indexes_orders", # noqa @@ -1508,6 +1571,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): "queries.tests.WeirdQuerysetSlicingTests.test_empty_sliced_subquery_exclude", # noqa "queries.tests.WeirdQuerysetSlicingTests.test_tickets_7698_10202", # noqa "queries.tests.WeirdQuerysetSlicingTests.test_zero_length_values_slicing", # noqa + "foreign_object.tests.MultiColumnFKTests.test_translations", # noqa "schema.tests.SchemaTests.test_add_datefield_and_datetimefield_use_effective_default", # noqa "schema.tests.SchemaTests.test_add_field", # noqa "schema.tests.SchemaTests.test_add_field_binary", # noqa @@ -1595,6 +1659,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): "select_related_onetoone.tests.ReverseSelectRelatedTestCase.test_nullable_relation", # noqa "select_related_onetoone.tests.ReverseSelectRelatedTestCase.test_self_relation", # noqa "sessions_tests.tests.CustomDatabaseSessionTests.test_actual_expiry", # noqa + "sessions_tests.tests.CustomDatabaseSessionTests.test_extra_session_field", # noqa "sessions_tests.tests.CustomDatabaseSessionTests.test_clearsessions_command", # noqa "sessions_tests.tests.CustomDatabaseSessionTests.test_cycle", # noqa "sessions_tests.tests.CustomDatabaseSessionTests.test_cycle_with_no_session_cache", # noqa @@ -1607,6 +1672,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): "sessions_tests.tests.CustomDatabaseSessionTests.test_session_save_does_not_resurrect_session_logged_out_in_other_context", # noqa "sessions_tests.tests.CustomDatabaseSessionTests.test_session_str", # noqa "sessions_tests.tests.CustomDatabaseSessionTests.test_sessionmanager_save", # noqa + "sessions_tests.tests.SessionMiddlewareTests.test_empty_session_saved", # noqa "sitemaps_tests.test_generic.GenericViewsSitemapTests.test_generic_sitemap", # noqa "sitemaps_tests.test_generic.GenericViewsSitemapTests.test_generic_sitemap_attributes", # noqa "sitemaps_tests.test_generic.GenericViewsSitemapTests.test_generic_sitemap_lastmod", # noqa @@ -1653,6 +1719,35 @@ class DatabaseFeatures(BaseDatabaseFeatures): "sitemaps_tests.test_utils.PingGoogleTests.test_get_sitemap_full_url_not_detected", # noqa "sitemaps_tests.test_utils.PingGoogleTests.test_something", # noqa "string_lookup.tests.StringLookupTests.test_queries_on_textfields", # noqa + "force_insert_update.tests.InheritanceTests.test_force_update_on_inherited_model_without_fields", # noqa + "force_insert_update.tests.InheritanceTests.test_force_update_on_inherited_model", # noqa + "admin_filters.tests.ListFiltersTests.test_relatedonlyfieldlistfilter_manytomany", # noqa + "admin_filters.tests.ListFiltersTests.test_relatedonlyfieldlistfilter_underscorelookup_foreignkey", # noqa + "admin_filters.tests.ListFiltersTests.test_simplelistfilter_with_none_returning_lookups", # noqa + "admin_filters.tests.ListFiltersTests.test_simplelistfilter_with_queryset_based_lookups", # noqa + "admin_filters.tests.ListFiltersTests.test_simplelistfilter_without_parameter", # noqa + "admin_filters.tests.ListFiltersTests.test_two_characters_long_field", # noqa + "fixtures.tests.FixtureLoadingTests.test_dumpdata_progressbar", # noqa + "fixtures.tests.FixtureLoadingTests.test_dumpdata_with_file_output", # noqa + "fixtures.tests.FixtureLoadingTests.test_dumpdata_with_pks", # noqa + "fixtures.tests.FixtureLoadingTests.test_loaddata_verbosity_three", # noqa + "fixtures.tests.FixtureLoadingTests.test_loading_and_dumping", # noqa + "fixtures.tests.FixtureLoadingTests.test_loading_stdin", # noqa + "fixtures.tests.FixtureLoadingTests.test_output_formats", # noqa + "fixtures.tests.TestCaseFixtureLoadingTests.testClassFixtures", # noqa + "fixtures_model_package.tests.FixtureTestCase.test_loaddata", # noqa + "fixtures_model_package.tests.SampleTestCase.testClassFixtures", # noqa + "get_or_create.tests.UpdateOrCreateTests.test_create_twice", # noqa + "get_or_create.tests.UpdateOrCreateTests.test_defaults_exact", # noqa + "get_or_create.tests.UpdateOrCreateTests.test_update", # noqa + "get_or_create.tests.UpdateOrCreateTests.test_update_callable_default", # noqa + "get_or_create.tests.UpdateOrCreateTests.test_update_with_many", # noqa + "get_or_create.tests.UpdateOrCreateTests.test_update_with_related_manager", # noqa + "model_inheritance_regress.tests.ModelInheritanceTest.test_model_inheritance", # noqa + "model_inheritance_regress.tests.ModelInheritanceTest.test_id_field_update_on_ancestor_change", # noqa + "model_inheritance.tests.ModelInheritanceTests.test_update_parent_filtering", # noqa + "model_inheritance.tests.ModelInheritanceDataTests.test_update_query_counts", # noqa + "model_inheritance.tests.ModelInheritanceDataTests.test_update_inherited_model", # noqa "test_client.tests.ClientTest.test_empty_post", # noqa "test_client.tests.ClientTest.test_exception_following_nested_client_request", # noqa "test_client.tests.ClientTest.test_external_redirect", # noqa diff --git a/django_test_apps.txt b/django_test_apps.txt index c4d41b5cf1..0c1674e8c1 100644 --- a/django_test_apps.txt +++ b/django_test_apps.txt @@ -1,138 +1,9 @@ -admin_changelist -admin_docs admin_filters -admin_inlines -admin_ordering admin_utils -admin_views -aggregation -aggregation_regress -annotations -auth_tests -backends -basic -bulk_create -cache -choices -constraints -contenttypes_tests -custom_columns custom_lookups -custom_managers -custom_methods -custom_pk -datatypes -dates -datetimes -db_functions -defer -defer_regress -delete -delete_regress -distinct_on_fields -empty -expressions -expressions_case -expressions_window extra_regress -field_defaults -file_storage -file_uploads -filtered_relation -fixtures -fixtures_model_package -fixtures_regress -flatpages_tests -force_insert_update -foreign_object -forms_tests -from_db_value -generic_inline_admin -generic_relations -generic_relations_regress -generic_views -get_earliest_or_latest -get_object_or_404 -get_or_create -i18n -indexes -inline_formsets -inspectdb -introspection -invalid_models_tests -known_related_objects -lookup -m2m_and_m2o -m2m_intermediary -m2m_multiple -m2m_recursive -m2m_regress -m2m_signals -m2m_through -m2m_through_regress -m2o_recursive -managers_regress -many_to_many -many_to_one -many_to_one_null -max_lengths -migrate_signals -migrations -migration_test_data_persistence -modeladmin -model_fields -model_forms -model_formsets -model_formsets_regress -model_indexes -model_inheritance -model_inheritance_regress -model_options -model_package +expressions_case model_regress -multiple_database -mutually_referential -nested_foreign_keys -null_fk -null_fk_ordering -null_queries -one_to_one -ordering -order_with_respect_to -or_lookups -prefetch_related -proxy_model_inheritance +modeladmin proxy_models -queries -queryset_pickle -raw_query -redirects_tests -reserved_names -reverse_lookup -save_delete_hooks -schema -select_for_update -select_related -select_related_onetoone -select_related_regress -serializers -servers -sessions_tests -signals -sitemaps_tests -sites_framework -sites_tests -string_lookup -syndication_tests -test_client -test_client_regress -test_runner -test_utils -timezones -transaction_hooks -transactions -unmanaged_models -update -update_only_fields -validation -view_tests +serializers \ No newline at end of file diff --git a/django_test_suite.sh b/django_test_suite.sh index e87cd6167f..fea10501e9 100755 --- a/django_test_suite.sh +++ b/django_test_suite.sh @@ -4,8 +4,31 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -# exit when any command fails -set -e +set -x pipefail + +sudo apt-get update -y +sudo apt-get install -y libmemcached-dev + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +export DJANGO_TESTS_DIR="django_tests_dir" +mkdir -p $DJANGO_TESTS_DIR + +if [ $SPANNER_EMULATOR_HOST != 0 ] +then + pip3 install . + pip3 install -e 'git+https://github.com/q-logic/python-spanner-django.git@dj_tests_against_emulator#egg=django-google-spanner' + git clone --depth 1 --single-branch --branch spanner-2.2.x https://github.com/timgraham/django.git $DJANGO_TESTS_DIR/django +fi + +# Install dependencies for Django tests. +sudo apt-get update +sudo apt-get install -y libffi-dev libjpeg-dev zlib1g-devel + +cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../ + +python3 create_test_instance.py # If no SPANNER_TEST_DB is set, generate a unique one # so that we can have multiple tests running without @@ -13,11 +36,13 @@ set -e # cleanup the created database. TEST_DBNAME=${SPANNER_TEST_DB:-$(python3 -c 'import os, time; print(chr(ord("a") + time.time_ns() % 26)+os.urandom(10).hex())')} TEST_DBNAME_OTHER="$TEST_DBNAME-ot" -TEST_APPS=${DJANGO_TEST_APPS:-basic} INSTANCE=${SPANNER_TEST_INSTANCE:-django-tests} -PROJECT=${PROJECT_ID:-appdev-soda-spanner-staging} +PROJECT=${PROJECT_ID} +SPANNER_EMULATOR_HOST=${SPANNER_EMULATOR_HOST} +GOOGLE_CLOUD_PROJECT=${GOOGLE_CLOUD_PROJECT} SETTINGS_FILE="$TEST_DBNAME-settings" TESTS_DIR=${DJANGO_TESTS_DIR:-django_tests} +TEST_APPS=${DJANGO_TEST_APPS} create_settings() { cat << ! > "$SETTINGS_FILE.py" @@ -42,21 +67,7 @@ PASSWORD_HASHERS = [ ! } -setup_emulator_if_needed() { - if [[ $SPANNER_EMULATOR_HOST != "" ]] - then - echo "Running the emulator at: $SPANNER_EMULATOR_HOST" - ./emulator_main --host_port "$SPANNER_EMULATOR_HOST" & - SPANNER_INSTANCE=$INSTANCE python3 .kokoro/ensure_instance_exists.py - fi -} - -run_django_tests() { - cd $TESTS_DIR/django/tests - create_settings - echo -e "\033[32mRunning Django tests: $TEST_APPS\033[00m" - python3 runtests.py $TEST_APPS --verbosity=2 --noinput --settings $SETTINGS_FILE -} +cd $TESTS_DIR/django/tests +create_settings -setup_emulator_if_needed -run_django_tests +python3 runtests.py $TEST_APPS --verbosity=3 --noinput --settings $SETTINGS_FILE \ No newline at end of file diff --git a/setup.py b/setup.py index d967e2ee32..5ca8570a3d 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ # 'Development Status :: 4 - Beta' # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 3 - Alpha" -dependencies = ["sqlparse >= 0.3.0", "google-cloud-spanner >= 2.1.0"] +dependencies = ["sqlparse >= 0.3.0", "google-cloud-spanner >= 3.0.0"] extras = {} BASE_DIR = os.path.dirname(__file__)