diff --git a/.github/workflows/django3.2_tests_against_emulator0.yml b/.github/workflows/django3.2_tests_against_emulator0.yml new file mode 100644 index 0000000000..79a8ed1966 --- /dev/null +++ b/.github/workflows/django3.2_tests_against_emulator0.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - main + pull_request: +name: django3.2-tests0 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite_3.2.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: admin_changelist admin_ordering aggregation distinct_on_fields expressions_window fixtures_model_package datetimes custom_methods generic_inline_admin field_defaults datatypes empty m2o_recursive many_to_one_null migration_test_data_persistence admin_docs invalid_models_tests migrate_signals model_forms.test_uuid model_forms.test_modelchoicefield syndication_tests view_tests update test_utils select_related_onetoone sessions_tests diff --git a/.github/workflows/django3.2_tests_against_emulator1.yml b/.github/workflows/django3.2_tests_against_emulator1.yml new file mode 100644 index 0000000000..b6b1da098d --- /dev/null +++ b/.github/workflows/django3.2_tests_against_emulator1.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - main + pull_request: +name: django3.2-tests1 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite_3.2.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: db_functions save_delete_hooks get_object_or_404 model_indexes custom_pk indexes transaction_hooks constraints schema custom_columns i18n from_db_value sites_tests mutually_referential model_package defer_regress update_only_fields backends redirects_tests expressions get_or_create foreign_object generic_relations_regress many_to_many select_related generic_relations queryset_pickle model_inheritance diff --git a/.github/workflows/django3.2_tests_against_emulator2.yml b/.github/workflows/django3.2_tests_against_emulator2.yml new file mode 100644 index 0000000000..adb8f7e436 --- /dev/null +++ b/.github/workflows/django3.2_tests_against_emulator2.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - main + pull_request: +name: django3.2-tests2 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite_3.2.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: model_options known_related_objects m2m_signals delete_regress fixtures generic_views model_inheritance_regress nested_foreign_keys lookup delete model_formsets diff --git a/.github/workflows/django3.2_tests_against_emulator3.yml b/.github/workflows/django3.2_tests_against_emulator3.yml new file mode 100644 index 0000000000..a71b8e2a3f --- /dev/null +++ b/.github/workflows/django3.2_tests_against_emulator3.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - main + pull_request: +name: django3.2-tests3 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite_3.2.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: signals or_lookups m2m_through_regress filtered_relation servers m2m_through fixtures_regress timezones model_forms.tests diff --git a/.github/workflows/django3.2_tests_against_emulator4.yml b/.github/workflows/django3.2_tests_against_emulator4.yml new file mode 100644 index 0000000000..cf9456f3ed --- /dev/null +++ b/.github/workflows/django3.2_tests_against_emulator4.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - main + pull_request: +name: django3.2-tests4 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite_3.2.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: introspection multiple_database null_fk_ordering ordering m2m_intermediary null_fk max_lengths dates force_insert_update test_client m2m_multiple test_client_regress sitemaps_tests admin_inlines transactions null_queries test_runner m2m_and_m2o prefetch_related m2m_regress file_uploads sites_framework auth_tests forms_tests inline_formsets order_with_respect_to contenttypes_tests defer diff --git a/.github/workflows/django3.2_tests_against_emulator5.yml b/.github/workflows/django3.2_tests_against_emulator5.yml new file mode 100644 index 0000000000..fcecc1219d --- /dev/null +++ b/.github/workflows/django3.2_tests_against_emulator5.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - main + pull_request: +name: django3.2-tests5 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite_3.2.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: file_storage m2m_recursive reverse_lookup managers_regress basic annotations unmanaged_models string_lookup aggregation_regress reserved_names select_for_update many_to_one cache select_related_regress flatpages_tests model_formsets_regress diff --git a/.github/workflows/django3.2_tests_against_emulator6.yml b/.github/workflows/django3.2_tests_against_emulator6.yml new file mode 100644 index 0000000000..8acc6ae53e --- /dev/null +++ b/.github/workflows/django3.2_tests_against_emulator6.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - main + pull_request: +name: django3.2-tests6 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite_3.2.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: model_fields queries.test_bulk_update queries.test_explain diff --git a/.github/workflows/django3.2_tests_against_emulator7.yml b/.github/workflows/django3.2_tests_against_emulator7.yml new file mode 100644 index 0000000000..c38f9045f0 --- /dev/null +++ b/.github/workflows/django3.2_tests_against_emulator7.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - main + pull_request: +name: django3.2-tests7 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite_3.2.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: queries.test_iterator queries.test_q queries.test_query queries.test_qs_combinators diff --git a/.github/workflows/django3.2_tests_against_emulator8.yml b/.github/workflows/django3.2_tests_against_emulator8.yml new file mode 100644 index 0000000000..b38f8488d9 --- /dev/null +++ b/.github/workflows/django3.2_tests_against_emulator8.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - main + pull_request: +name: django3.2-tests8 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite_3.2.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: inspectdb custom_managers migrations validation get_earliest_or_latest proxy_model_inheritance one_to_one raw_query bulk_create diff --git a/.github/workflows/django3.2_tests_against_emulator9.yml b/.github/workflows/django3.2_tests_against_emulator9.yml new file mode 100644 index 0000000000..f55683bed9 --- /dev/null +++ b/.github/workflows/django3.2_tests_against_emulator9.yml @@ -0,0 +1,32 @@ +on: + push: + branches: + - main + pull_request: +name: django3.2-tests9 +jobs: + system-tests: + runs-on: ubuntu-latest + + services: + emulator-0: + image: gcr.io/cloud-spanner-emulator/emulator:latest + ports: + - 9010:9010 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Run Django tests + run: sh django_test_suite_3.2.sh + env: + SPANNER_EMULATOR_HOST: localhost:9010 + GOOGLE_CLOUD_PROJECT: emulator-test-project + GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE: true + RUNNING_SPANNER_BACKEND_TESTS: 1 + SPANNER_TEST_INSTANCE: google-cloud-django-backend-tests + DJANGO_TEST_APPS: queries.tests diff --git a/.github/workflows/integration-tests-against-emulator.yml b/.github/workflows/integration-tests-against-emulator.yml index 5767e966bb..3e8044bc90 100644 --- a/.github/workflows/integration-tests-against-emulator.yml +++ b/.github/workflows/integration-tests-against-emulator.yml @@ -7,6 +7,9 @@ name: Run Django Spanner integration tests against emulator jobs: system-tests: runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] services: emulator: @@ -18,10 +21,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Setup Python + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: ${{ matrix.python-version }} - name: Install nox run: python -m pip install nox - name: Run nox diff --git a/README.rst b/README.rst index 62739a3182..4f0d9ca886 100644 --- a/README.rst +++ b/README.rst @@ -253,118 +253,11 @@ Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See the `Code of Conduct `_ for more information. -Current limitations -------------------- -``AutoField`` generates random IDs -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Spanner doesn't have support for auto-generating primary key values. -Therefore, ``django-google-spanner`` monkey-patches ``AutoField`` to generate a -random UUID4. It generates a default using ``Field``'s ``default`` option which -means ``AutoField``\ s will have a value when a model instance is created. For -example: - -:: - - >>> ExampleModel() - >>> ExampleModel.pk - 4229421414948291880 - -To avoid -`hotspotting `__, -these IDs are not monotonically increasing. This means that sorting -models by ID isn't guaranteed to return them in the order in which they -were created. - -``ForeignKey`` constraints aren't created (`#313 `__) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Spanner does not support ``ON DELETE CASCADE`` when creating foreign-key -constraints, so this is not supported in ``django-google-spanner``. - -``Unsigned`` datatypes are not supported -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Spanner does not support ``Unsigned`` datatypes so `PositiveIntegerField -`__ -and `PositiveSmallIntegerField -`__ -are both stored as `Integer type -`__ -. - -``Meta.order_with_respect_to`` model option isn't supported -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This feature uses a column name that starts with an underscore -(``_order``) which Spanner doesn't allow. - -Random ``QuerySet`` ordering isn't supported -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Spanner does not support it and will throw an exception. For example: - -:: - - >>> ExampleModel.objects.order_by('?') - ... - django.db.utils.ProgrammingError: 400 Function not found: RANDOM ... FROM - example_model ORDER BY RANDOM() ASC - -Schema migrations -~~~~~~~~~~~~~~~~~ - -There are some limitations on schema changes to consider: - -- No support for renaming tables and columns; -- A column's type can't be changed; -- A table's primary key can't be altered. - -``DurationField`` arithmetic doesn't work with ``DateField`` values (`#253 `__) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Spanner requires using different functions for arithmetic depending on -the column type: - -- ``TIMESTAMP`` columns (``DateTimeField``) require ``TIMESTAMP_ADD`` - or ``TIMESTAMP_SUB`` -- ``DATE`` columns (``DateField``) require ``DATE_ADD`` or ``DATE_SUB`` - -Django does not provide ways to determine which database function to -use. ``DatabaseOperations.combine_duration_expression()`` arbitrarily uses -``TIMESTAMP_ADD`` and ``TIMESTAMP_SUB``. Therefore, if you use a -``DateField`` in a ``DurationField`` expression, you'll likely see an error -such as: - -:: - - "No matching signature for function TIMESTAMP\_ADD for argument types: - DATE, INTERVAL INT64 DATE\_TIME\_PART." - -Computations that yield FLOAT64 values cannot be assigned to INT64 columns -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Spanner does not support this (`#331 -`__) and will -throw an error: - -:: - - >>> ExampleModel.objects.update(integer=F('integer') / 2) - ... - django.db.utils.ProgrammingError: 400 Value of type FLOAT64 cannot be - assigned to integer, which has type INT64 [at 1:46]\nUPDATE - example_model SET integer = (example_model.integer /... - -Addition with null values crash -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Additions cannot include ``None`` values. For example: +LIMITATIONS +----------------- -:: +Spanner has certain limitations of it's own and a full set of limitations are documented over `here `_ +It is recommended that you go through that list. - >>> Book.objects.annotate(adjusted_rating=F('rating') + None) - ... - google.api_core.exceptions.InvalidArgument: 400 Operands of + cannot be literal - NULL ... +Django spanner has a set of limitations as well, please go through the `list `_. diff --git a/django_spanner/__init__.py b/django_spanner/__init__.py index a26703d5a5..54c513d60d 100644 --- a/django_spanner/__init__.py +++ b/django_spanner/__init__.py @@ -6,29 +6,49 @@ import datetime import os +import django # Monkey-patch AutoField to generate a random value since Cloud Spanner can't # do that. from uuid import uuid4 import pkg_resources -from django.db.models.fields import AutoField, Field - -# Monkey-patch google.DatetimeWithNanoseconds's __eq__ compare against -# datetime.datetime. -from google.api_core.datetime_helpers import DatetimeWithNanoseconds +from google.cloud.spanner_v1 import JsonObject +from django.db.models.fields import ( + AutoField, + Field, +) from .expressions import register_expressions from .functions import register_functions from .lookups import register_lookups from .utils import check_django_compatability +# Monkey-patch google.DatetimeWithNanoseconds's __eq__ compare against +# datetime.datetime. +from google.api_core.datetime_helpers import DatetimeWithNanoseconds + + +USING_DJANGO_3 = False +if django.VERSION[:2] == (3, 2): + USING_DJANGO_3 = True + +if USING_DJANGO_3: + from django.db.models.fields import ( + SmallAutoField, + BigAutoField, + ) + from django.db.models import JSONField + __version__ = pkg_resources.get_distribution("django-google-spanner").version USE_EMULATOR = os.getenv("SPANNER_EMULATOR_HOST") is not None -check_django_compatability() -register_expressions() +# Only active LTS django versions (2.2.*, 3.2.*) are supported by this library right now. +SUPPORTED_DJANGO_VERSIONS = [(2, 2), (3, 2)] + +check_django_compatability(SUPPORTED_DJANGO_VERSIONS) +register_expressions(USING_DJANGO_3) register_functions() register_lookups() @@ -45,6 +65,25 @@ def autofield_init(self, *args, **kwargs): AutoField.__init__ = autofield_init +AutoField.db_returning = False +AutoField.validators = [] +if USING_DJANGO_3: + SmallAutoField.__init__ = autofield_init + BigAutoField.__init__ = autofield_init + SmallAutoField.db_returning = False + BigAutoField.db_returning = False + SmallAutoField.validators = [] + BigAutoField.validators = [] + + def get_prep_value(self, value): + # Json encoding and decoding for spanner is done in python-spanner. + if not isinstance(value, JsonObject) and isinstance(value, dict): + return JsonObject(value) + + return value + + JSONField.get_prep_value = get_prep_value + old_datetimewithnanoseconds_eq = getattr( DatetimeWithNanoseconds, "__eq__", None diff --git a/django_spanner/base.py b/django_spanner/base.py index 4a4b86ff7d..25c42416a5 100644 --- a/django_spanner/base.py +++ b/django_spanner/base.py @@ -34,6 +34,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): "DateField": "DATE", "DateTimeField": "TIMESTAMP", "DecimalField": "NUMERIC", + "JSONField": "JSON", "DurationField": "INT64", "EmailField": "STRING(%(max_length)s)", "FileField": "STRING(%(max_length)s)", @@ -45,6 +46,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): "GenericIPAddressField": "STRING(39)", "NullBooleanField": "BOOL", "OneToOneField": "INT64", + "PositiveBigIntegerField": "INT64", "PositiveIntegerField": "INT64", "PositiveSmallIntegerField": "INT64", "SlugField": "STRING(%(max_length)s)", @@ -96,6 +98,12 @@ class DatabaseWrapper(BaseDatabaseWrapper): "iendswith": "", } + data_type_check_constraints = { + "PositiveBigIntegerField": "%(column)s >= 0", + "PositiveIntegerField": "%(column)s >= 0", + "PositiveSmallIntegerField": "%(column)s >= 0", + } + Database = spanner_dbapi SchemaEditorClass = DatabaseSchemaEditor creation_class = DatabaseCreation diff --git a/django_spanner/expressions.py b/django_spanner/expressions.py index 526b6a7362..44a90f586d 100644 --- a/django_spanner/expressions.py +++ b/django_spanner/expressions.py @@ -15,9 +15,6 @@ def order_by(self, compiler, connection, **extra_context): :rtype: str :returns: A SQL query. """ - # TODO: In Django 3.1, this can be replaced with - # DatabaseFeatures.supports_order_by_nulls_modifier = False. - # Also, consider making this a class method. template = None if self.nulls_last: template = "%(expression)s IS NULL, %(expression)s %(ordering)s" @@ -28,6 +25,9 @@ def order_by(self, compiler, connection, **extra_context): ) -def register_expressions(): - """Add Spanner-specific attribute to the Django OrderBy class.""" - OrderBy.as_spanner = order_by +def register_expressions(using_django_3=False): + """Add Spanner-specific attribute to the Django OrderBy class for django 2.2.""" + # In Django >= 3.1, this can be replaced with + # DatabaseFeatures.supports_order_by_nulls_modifier = False. + if not using_django_3: + OrderBy.as_spanner = order_by diff --git a/django_spanner/features.py b/django_spanner/features.py index a0ae6299c3..87abddf87f 100644 --- a/django_spanner/features.py +++ b/django_spanner/features.py @@ -8,6 +8,7 @@ from django.db.backends.base.features import BaseDatabaseFeatures from django.db.utils import InterfaceError +from django_spanner import USE_EMULATOR, USING_DJANGO_3 class DatabaseFeatures(BaseDatabaseFeatures): @@ -23,6 +24,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): # https://cloud.google.com/spanner/quotas#query_limits max_query_params = 900 supports_foreign_keys = False + can_create_inline_fk = False supports_ignore_conflicts = False supports_partial_indexes = False supports_regex_backreferencing = False @@ -30,9 +32,32 @@ class DatabaseFeatures(BaseDatabaseFeatures): supports_sequence_reset = False supports_timezones = False supports_transactions = False - supports_column_check_constraints = False - supports_table_check_constraints = False + if USE_EMULATOR: + # Emulator does not support json. + supports_json_field = False + # Emulator does not support check constrints. + supports_column_check_constraints = False + supports_table_check_constraints = False + else: + supports_column_check_constraints = True + supports_table_check_constraints = True + if USING_DJANGO_3: + supports_json_field = True + else: + # Since JsonField was introduced in django3.1 we don't support it for django 2.2 + supports_json_field = False + supports_primitives_in_json_field = False + # Spanner does not support order by null modifiers. + # For Django 2.2 this feature is handled in code. + if USING_DJANGO_3: + supports_order_by_nulls_modifier = False + # Spanner does not support SELECTing an arbitrary expression that also + # appears in the GROUP BY clause. + supports_subqueries_in_group_by = False uses_savepoints = False + # Spanner does not support expression indexes + # example: CREATE INDEX index_name ON table (LOWER(column_name)) + supports_expression_indexes = False # Django tests that aren't supported by Spanner. skip_tests = ( @@ -151,12 +176,9 @@ class DatabaseFeatures(BaseDatabaseFeatures): "prefetch_related.test_prefetch_related_objects.PrefetchRelatedObjectsTests.test_m2m_then_m2m", "prefetch_related.tests.CustomPrefetchTests.test_custom_qs", "prefetch_related.tests.CustomPrefetchTests.test_nested_prefetch_related_are_not_overwritten", - "prefetch_related.tests.DirectPrefechedObjectCacheReuseTests.test_detect_is_fetched", - "prefetch_related.tests.DirectPrefechedObjectCacheReuseTests.test_detect_is_fetched_with_to_attr", "prefetch_related.tests.ForeignKeyToFieldTest.test_m2m", "queries.test_bulk_update.BulkUpdateNoteTests.test_multiple_fields", "queries.test_bulk_update.BulkUpdateTests.test_inherited_fields", - "queries.tests.Queries1Tests.test_ticket9411", "queries.tests.Queries4Tests.test_ticket15316_exclude_true", "queries.tests.Queries5Tests.test_ticket7256", "queries.tests.SubqueryTests.test_related_sliced_subquery", @@ -179,17 +201,12 @@ class DatabaseFeatures(BaseDatabaseFeatures): "timezones.tests.NewDatabaseTests.test_query_datetimes", # using NULL with + crashes: https://github.com/googleapis/python-spanner-django/issues/201 "annotations.tests.NonAggregateAnnotationTestCase.test_combined_annotation_commutative", - # Spanner loses DecimalField precision due to conversion to float: - # https://github.com/googleapis/python-spanner-django/pull/133#pullrequestreview-328482925 - "aggregation.tests.AggregateTestCase.test_decimal_max_digits_has_no_effect", - "aggregation.tests.AggregateTestCase.test_related_aggregate", + # Spanner does not support custom precision on DecimalField "db_functions.comparison.test_cast.CastTests.test_cast_to_decimal_field", "model_fields.test_decimalfield.DecimalFieldTests.test_fetch_from_db_without_float_rounding", "model_fields.test_decimalfield.DecimalFieldTests.test_roundtrip_with_trailing_zeros", - # Spanner does not support unsigned integer field. - "model_fields.test_integerfield.PositiveIntegerFieldTests.test_negative_values", # Spanner doesn't support the variance the standard deviation database - # functions: + # functions on full population. "aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_numerical_aggregates", "aggregation_regress.tests.AggregationTests.test_stddev", # SELECT list expression references which is neither grouped @@ -240,11 +257,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "db_functions.comparison.test_cast.CastTests.test_cast_from_db_date_to_datetime", # Tests that fail during tear down on databases that don't support # transactions: https://github.com/googleapis/python-spanner-django/issues/271 - "admin_views.test_multidb.MultiDatabaseTests.test_add_view", - "admin_views.test_multidb.MultiDatabaseTests.test_change_view", - "admin_views.test_multidb.MultiDatabaseTests.test_delete_view", - "auth_tests.test_admin_multidb.MultiDatabaseTests.test_add_view", - "auth_tests.test_remote_user_deprecation.RemoteUserCustomTest.test_configure_user_deprecation_warning", "contenttypes_tests.test_models.ContentTypesMultidbTests.test_multidb", # Tests that by-pass using django_spanner and generate # invalid DDL: https://github.com/googleapis/python-spanner-django/issues/298 @@ -269,9 +281,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "transaction_hooks.tests.TestConnectionOnCommit.test_discards_hooks_from_rolled_back_savepoint", "transaction_hooks.tests.TestConnectionOnCommit.test_inner_savepoint_rolled_back_with_outer", "transaction_hooks.tests.TestConnectionOnCommit.test_inner_savepoint_does_not_affect_outer", - # Spanner doesn't support views. - "inspectdb.tests.InspectDBTransactionalTests.test_include_views", - "introspection.tests.IntrospectionTests.test_table_names_with_views", # No sequence for AutoField in Spanner. "introspection.tests.IntrospectionTests.test_sequence_list", # DatabaseIntrospection.get_key_columns() is only required if this @@ -375,12 +384,136 @@ class DatabaseFeatures(BaseDatabaseFeatures): "view_tests.tests.test_csrf.CsrfViewTests.test_no_referer", "view_tests.tests.test_i18n.SetLanguageTests.test_lang_from_translated_i18n_pattern", ) + if USING_DJANGO_3: + skip_tests += ( + # Spanner does not support UUID field natively + "model_fields.test_uuid.TestQuerying.test_iexact", + # Spanner does not support setting a default value on columns. + "schema.tests.SchemaTests.test_alter_text_field_to_not_null_with_default_value", + # Direct SQL query test that do not follow spanner syntax. + "schema.tests.SchemaTests.test_alter_auto_field_quoted_db_column", + "schema.tests.SchemaTests.test_alter_primary_key_quoted_db_table", + # Insert sql with param variables using %(name)s parameter style is failing + # https://github.com/googleapis/python-spanner/issues/542 + "backends.tests.LastExecutedQueryTest.test_last_executed_query_dict", + # Spanner autofield is replaced with uuid4 so validation is disabled + "model_fields.test_autofield.AutoFieldTests.test_backend_range_validation", + "model_fields.test_autofield.AutoFieldTests.test_redundant_backend_range_validators", + "model_fields.test_autofield.AutoFieldTests.test_redundant_backend_range_validators", + "model_fields.test_autofield.BigAutoFieldTests.test_backend_range_validation", + "model_fields.test_autofield.BigAutoFieldTests.test_redundant_backend_range_validators", + "model_fields.test_autofield.BigAutoFieldTests.test_redundant_backend_range_validators", + "model_fields.test_autofield.SmallAutoFieldTests.test_backend_range_validation", + "model_fields.test_autofield.SmallAutoFieldTests.test_redundant_backend_range_validators", + "model_fields.test_autofield.SmallAutoFieldTests.test_redundant_backend_range_validators", + # Spanner does not support deferred unique constraints + "migrations.test_operations.OperationTests.test_create_model_with_deferred_unique_constraint", + # Spanner does not support JSON object query on fields. + "db_functions.comparison.test_json_object.JSONObjectTests.test_empty", + "db_functions.comparison.test_json_object.JSONObjectTests.test_basic", + "db_functions.comparison.test_json_object.JSONObjectTests.test_expressions", + "db_functions.comparison.test_json_object.JSONObjectTests.test_nested_empty_json_object", + "db_functions.comparison.test_json_object.JSONObjectTests.test_nested_json_object", + "db_functions.comparison.test_json_object.JSONObjectTests.test_textfield", + # Spanner does not support iso_week_day but week_day is supported. + "timezones.tests.LegacyDatabaseTests.test_query_datetime_lookups", + "timezones.tests.NewDatabaseTests.test_query_datetime_lookups", + "timezones.tests.NewDatabaseTests.test_query_datetime_lookups_in_other_timezone", + "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_func", + "db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_iso_weekday_func", + "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func", + "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func_with_timezone", + "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func_with_timezone", + "db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_iso_weekday_func", + # Spanner gived SHA encryption output in bytes, django expects it in hex string format. + "db_functions.text.test_sha512.SHA512Tests.test_basic", + "db_functions.text.test_sha512.SHA512Tests.test_transform", + "db_functions.text.test_md5.MD5Tests.test_basic", + "db_functions.text.test_md5.MD5Tests.test_transform", + "db_functions.text.test_sha1.SHA1Tests.test_basic", + "db_functions.text.test_sha1.SHA1Tests.test_transform", + "db_functions.text.test_sha224.SHA224Tests.test_basic", + "db_functions.text.test_sha224.SHA224Tests.test_transform", + "db_functions.text.test_sha256.SHA256Tests.test_basic", + "db_functions.text.test_sha256.SHA256Tests.test_transform", + "db_functions.text.test_sha384.SHA384Tests.test_basic", + "db_functions.text.test_sha384.SHA384Tests.test_transform", + # Spanner does not support RANDOM number generation function + "db_functions.math.test_random.RandomTests.test", + # Spanner supports order by id, but it's does not work the same way as + # an auto increment field. + "model_forms.test_modelchoicefield.ModelChoiceFieldTests.test_choice_iterator_passes_model_to_widget", + "queries.test_qs_combinators.QuerySetSetOperationTests.test_union_with_values_list_and_order", + "ordering.tests.OrderingTests.test_order_by_self_referential_fk", + "fixtures.tests.ForwardReferenceTests.test_forward_reference_m2m_natural_key", + "fixtures.tests.ForwardReferenceTests.test_forward_reference_fk_natural_key", + # Spanner does not support empty list of DML statement. + "backends.tests.BackendTestCase.test_cursor_executemany_with_empty_params_list", + # Spanner does not support SELECTing an arbitrary expression that also + # appears in the GROUP BY clause. + "annotations.tests.NonAggregateAnnotationTestCase.test_grouping_by_q_expression_annotation", + # No foreign key constraints in Spanner. + "backends.tests.FkConstraintsTests.test_check_constraints_sql_keywords", + # No Django transaction management in Spanner. + "transactions.tests.DisableDurabiltityCheckTests.test_nested_both_durable", + "transactions.tests.DisableDurabiltityCheckTests.test_nested_inner_durable", + # Tests that expect it to be empty untill saved in db. + "test_utils.test_testcase.TestDataTests.test_class_attribute_identity", + "model_fields.test_jsonfield.TestSerialization.test_dumping", + "model_fields.test_jsonfield.TestSerialization.test_dumping", + "model_fields.test_jsonfield.TestSerialization.test_dumping", + "model_fields.test_jsonfield.TestSerialization.test_xml_serialization", + "model_fields.test_jsonfield.TestSerialization.test_xml_serialization", + "model_fields.test_jsonfield.TestSerialization.test_xml_serialization", + "bulk_create.tests.BulkCreateTests.test_unsaved_parent", + # Tests that assume a serial pk. + "lookup.tests.LookupTests.test_exact_query_rhs_with_selected_columns", + "prefetch_related.tests.DirectPrefetchedObjectCacheReuseTests.test_detect_is_fetched", + "prefetch_related.tests.DirectPrefetchedObjectCacheReuseTests.test_detect_is_fetched_with_to_attr", + # datetimes retrieved from the database with the wrong hour when + # USE_TZ = True: https://github.com/googleapis/python-spanner-django/issues/193 + "timezones.tests.NewDatabaseTests.test_query_convert_timezones", + # Spanner doesn't support random ordering. + "aggregation.tests.AggregateTestCase.test_aggregation_random_ordering", + # Tests that require transactions. + "test_utils.tests.CaptureOnCommitCallbacksTests.test_execute", + "test_utils.tests.CaptureOnCommitCallbacksTests.test_no_arguments", + "test_utils.tests.CaptureOnCommitCallbacksTests.test_pre_callback", + "test_utils.tests.CaptureOnCommitCallbacksTests.test_using", + # Field: GenericIPAddressField is mapped to String in Spanner + "inspectdb.tests.InspectDBTestCase.test_field_types", + # BigIntegerField is mapped to IntegerField in Spanner + "inspectdb.tests.InspectDBTestCase.test_number_field_types", + # Spanner limitation: Cannot change type of column. + "schema.tests.SchemaTests.test_char_field_pk_to_auto_field", + "schema.tests.SchemaTests.test_ci_cs_db_collation", + # Spanner limitation: Cannot rename tables and columns. + "migrations.test_operations.OperationTests.test_rename_field_case", + # Tests that sometimes fail on Kokoro for unknown reasons. + "migrations.test_operations.OperationTests.test_add_constraint_combinable", + # Tests that fail but are not related to spanner. + "test_utils.test_testcase.TestDataTests.test_undeepcopyable_warning", + ) + else: + # Tests specific to django 2.2 + skip_tests += ( + # Tests that assume a serial pk. + "prefetch_related.tests.DirectPrefechedObjectCacheReuseTests.test_detect_is_fetched", + "prefetch_related.tests.DirectPrefechedObjectCacheReuseTests.test_detect_is_fetched_with_to_attr", + ) if os.environ.get("SPANNER_EMULATOR_HOST", None): # Some code isn't yet supported by the Spanner emulator. skip_tests += ( + # Emulator doesn't support views. + "inspectdb.tests.InspectDBTransactionalTests.test_include_views", + "introspection.tests.IntrospectionTests.test_table_names_with_views", + # Check constraints are not supported by Spanner emulator. + "constraints.tests.CheckConstraintTests.test_database_constraint", # noqa + "constraints.tests.CheckConstraintTests.test_name", # noqa # Untyped parameters are not supported: # https://github.com/GoogleCloudPlatform/cloud-spanner-emulator#features-and-limitations + "auth_tests.test_admin_multidb.MultiDatabaseTests.test_add_view", # noqa "admin_changelist.test_date_hierarchy.DateHierarchyTests.test_bounded_params", # noqa "admin_changelist.test_date_hierarchy.DateHierarchyTests.test_bounded_params_with_time_zone", # noqa "admin_changelist.test_date_hierarchy.DateHierarchyTests.test_invalid_params", # noqa @@ -392,14 +525,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "admin_changelist.tests.ChangeListTests.test_custom_paginator", # noqa "admin_changelist.tests.ChangeListTests.test_deterministic_order_for_model_ordered_by_its_manager", # noqa "admin_changelist.tests.ChangeListTests.test_deterministic_order_for_unordered_model", # noqa - "admin_changelist.tests.ChangeListTests.test_distinct_for_inherited_m2m_in_list_filter", # noqa - "admin_changelist.tests.ChangeListTests.test_distinct_for_m2m_in_list_filter", # noqa - "admin_changelist.tests.ChangeListTests.test_distinct_for_m2m_to_inherited_in_list_filter", # noqa - "admin_changelist.tests.ChangeListTests.test_distinct_for_many_to_many_at_second_level_in_search_fields", # noqa - "admin_changelist.tests.ChangeListTests.test_distinct_for_non_unique_related_object_in_list_filter", # noqa - "admin_changelist.tests.ChangeListTests.test_distinct_for_non_unique_related_object_in_search_fields", # noqa - "admin_changelist.tests.ChangeListTests.test_distinct_for_through_m2m_at_second_level_in_list_filter", # noqa - "admin_changelist.tests.ChangeListTests.test_distinct_for_through_m2m_in_list_filter", # noqa "admin_changelist.tests.ChangeListTests.test_dynamic_list_display", # noqa "admin_changelist.tests.ChangeListTests.test_dynamic_list_display_links", # noqa "admin_changelist.tests.ChangeListTests.test_dynamic_list_filter", # noqa @@ -409,7 +534,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "admin_changelist.tests.ChangeListTests.test_get_list_editable_queryset_with_regex_chars_in_prefix", # noqa "admin_changelist.tests.ChangeListTests.test_get_select_related_custom_method", # noqa "admin_changelist.tests.ChangeListTests.test_multiuser_edit", # noqa - "admin_changelist.tests.ChangeListTests.test_no_distinct_for_m2m_in_list_filter_without_params", # noqa "admin_changelist.tests.ChangeListTests.test_no_list_display_links", # noqa "admin_changelist.tests.ChangeListTests.test_object_tools_displayed_no_add_permission", # noqa "admin_changelist.tests.ChangeListTests.test_pagination", # noqa @@ -572,7 +696,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "aggregation.tests.AggregateTestCase.test_filter_aggregate", # noqa "aggregation.tests.AggregateTestCase.test_fkey_aggregate", # noqa "aggregation.tests.AggregateTestCase.test_grouped_annotation_in_group_by", # noqa - "aggregation.tests.AggregateTestCase.test_missing_output_field_raises_error", # noqa "aggregation.tests.AggregateTestCase.test_more_aggregation", # noqa "aggregation.tests.AggregateTestCase.test_multi_arg_aggregate", # noqa "aggregation.tests.AggregateTestCase.test_multiple_aggregates", # noqa @@ -651,10 +774,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "auth_tests.test_context_processors.AuthContextProcessorTests.test_session_is_accessed", # noqa "auth_tests.test_context_processors.AuthContextProcessorTests.test_session_not_accessed", # noqa "auth_tests.test_context_processors.AuthContextProcessorTests.test_user_attrs", # noqa - "auth_tests.test_decorators.LoginRequiredTestCase.testCallable", # noqa - "auth_tests.test_decorators.LoginRequiredTestCase.testLoginRequired", # noqa - "auth_tests.test_decorators.LoginRequiredTestCase.testLoginRequiredNextUrl", # noqa - "auth_tests.test_decorators.LoginRequiredTestCase.testView", # noqa "auth_tests.test_decorators.PermissionsRequiredDecoratorTest.test_many_permissions_in_set_pass", # noqa "auth_tests.test_decorators.PermissionsRequiredDecoratorTest.test_many_permissions_pass", # noqa "auth_tests.test_decorators.PermissionsRequiredDecoratorTest.test_permissioned_denied_exception_raised", # noqa @@ -826,20 +945,12 @@ class DatabaseFeatures(BaseDatabaseFeatures): "auth_tests.test_remote_user.RemoteUserTest.test_last_login", # noqa "auth_tests.test_remote_user.RemoteUserTest.test_unknown_user", # noqa "auth_tests.test_remote_user.RemoteUserTest.test_user_switch_forces_new_login", # noqa - "auth_tests.test_remote_user_deprecation.RemoteUserCustomTest.test_configure_user_deprecation_warning", # noqa "auth_tests.test_signals.SignalTestCase.test_failed_login_without_request", # noqa "auth_tests.test_signals.SignalTestCase.test_login", # noqa "auth_tests.test_signals.SignalTestCase.test_login_with_custom_user_without_last_login_field", # noqa "auth_tests.test_signals.SignalTestCase.test_logout", # noqa "auth_tests.test_signals.SignalTestCase.test_logout_anonymous", # noqa "auth_tests.test_signals.SignalTestCase.test_update_last_login", # noqa - "auth_tests.test_templates.AuthTemplateTests.test_PasswordChangeDoneView", # noqa - "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetChangeView", # noqa - "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetCompleteView", # noqa - "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetConfirmView_invalid_token", # noqa - "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetConfirmView_valid_token", # noqa - "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetDoneView", # noqa - "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetView", # noqa "auth_tests.test_tokens.TokenGeneratorTest.test_10265", # noqa "auth_tests.test_tokens.TokenGeneratorTest.test_check_token_with_nonexistent_token_and_user", # noqa "auth_tests.test_tokens.TokenGeneratorTest.test_make_token", # noqa @@ -1172,7 +1283,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "generic_inline_admin.tests.GenericAdminViewTest.test_basic_add_POST", # noqa "generic_inline_admin.tests.GenericAdminViewTest.test_basic_edit_GET", # noqa "generic_inline_admin.tests.GenericAdminViewTest.test_basic_edit_POST", # noqa - "generic_inline_admin.tests.GenericInlineAdminParametersTest.testMaxNumParam", # noqa "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_get_extra", # noqa "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_extra_param", # noqa "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_get_max_num", # noqa @@ -1288,7 +1398,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "migrations.test_operations.OperationTests.test_alter_model_table_noop", # noqa "migrations.test_operations.OperationTests.test_alter_unique_together", # noqa "migrations.test_operations.OperationTests.test_alter_unique_together_remove", # noqa - "migrations.test_operations.OperationTests.test_autofield_foreignfield_growth", # noqa "migrations.test_operations.OperationTests.test_column_name_quoting", # noqa "migrations.test_operations.OperationTests.test_create_model", # noqa "migrations.test_operations.OperationTests.test_create_model_inheritance", # noqa @@ -1395,7 +1504,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "null_queries.tests.NullQueriesTests.test_reverse_relations", # noqa "ordering.tests.OrderingTests.test_default_ordering", # noqa "ordering.tests.OrderingTests.test_default_ordering_override", # noqa - "ordering.tests.OrderingTests.test_deprecated_values_annotate", # noqa "ordering.tests.OrderingTests.test_extra_ordering", # noqa "ordering.tests.OrderingTests.test_extra_ordering_quoting", # noqa "ordering.tests.OrderingTests.test_extra_ordering_with_table_name", # noqa @@ -1471,7 +1579,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "queries.tests.Queries1Tests.test_ticket2306", # noqa "queries.tests.Queries1Tests.test_ticket2400", # noqa "queries.tests.Queries1Tests.test_ticket2496", # noqa - "queries.tests.Queries1Tests.test_ticket2902", # noqa "queries.tests.Queries1Tests.test_ticket3037", # noqa "queries.tests.Queries1Tests.test_ticket3141", # noqa "queries.tests.Queries1Tests.test_ticket4358", # noqa @@ -1600,7 +1707,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "schema.tests.SchemaTests.test_alter_auto_field_to_integer_field", # noqa "schema.tests.SchemaTests.test_alter_charfield_to_null", # noqa "schema.tests.SchemaTests.test_alter_field_add_index_to_integerfield", # noqa - "schema.tests.SchemaTests.test_alter_field_default_doesnt_perfom_queries", # noqa "schema.tests.SchemaTests.test_alter_field_default_dropped", # noqa "schema.tests.SchemaTests.test_alter_field_fk_keeps_index", # noqa "schema.tests.SchemaTests.test_alter_field_fk_to_o2o", # noqa @@ -1695,7 +1801,6 @@ class DatabaseFeatures(BaseDatabaseFeatures): "sitemaps_tests.test_http.HTTPSitemapTests.test_paged_sitemap", # noqa "sitemaps_tests.test_http.HTTPSitemapTests.test_requestsite_sitemap", # noqa "sitemaps_tests.test_http.HTTPSitemapTests.test_simple_custom_sitemap", # noqa - "sitemaps_tests.test_http.HTTPSitemapTests.test_simple_i18nsitemap_index", # noqa "sitemaps_tests.test_http.HTTPSitemapTests.test_simple_sitemap", # noqa "sitemaps_tests.test_http.HTTPSitemapTests.test_simple_sitemap_custom_index", # noqa "sitemaps_tests.test_http.HTTPSitemapTests.test_simple_sitemap_index", # noqa @@ -1744,9 +1849,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): "fixtures.tests.FixtureLoadingTests.test_loading_and_dumping", # noqa "fixtures.tests.FixtureLoadingTests.test_loading_stdin", # noqa "fixtures.tests.FixtureLoadingTests.test_output_formats", # noqa - "fixtures.tests.TestCaseFixtureLoadingTests.testClassFixtures", # noqa "fixtures_model_package.tests.FixtureTestCase.test_loaddata", # noqa - "fixtures_model_package.tests.SampleTestCase.testClassFixtures", # noqa "get_or_create.tests.UpdateOrCreateTests.test_create_twice", # noqa "get_or_create.tests.UpdateOrCreateTests.test_defaults_exact", # noqa "get_or_create.tests.UpdateOrCreateTests.test_update", # noqa @@ -1879,3 +1982,164 @@ class DatabaseFeatures(BaseDatabaseFeatures): "validation.tests.GenericIPAddressFieldTests.test_v4_unpack_uniqueness_detection", # noqa "validation.tests.GenericIPAddressFieldTests.test_v6_uniqueness_detection", # noqa ) + + if USING_DJANGO_3: + # Some tests are different between django 3.2 and 2.2. + skip_tests += ( + # Check constraints are not supported by Spanner emulator. + "constraints.tests.CheckConstraintTests.test_abstract_name", # noqa + "constraints.tests.CheckConstraintTests.test_database_constraint_expression", # noqa + "constraints.tests.CheckConstraintTests.test_database_constraint_expressionwrapper", # noqa + "constraints.tests.CheckConstraintTests.test_database_constraint_unicode", # noqa + # Untyped parameters are not supported: + # https://github.com/GoogleCloudPlatform/cloud-spanner-emulator#features-and-limitations + "admin_changelist.test_date_hierarchy.DateHierarchyTests.test_bounded_params_with_dst_time_zone", # noqa + "admin_changelist.tests.ChangeListTests.test_changelist_search_form_validation", # noqa + "admin_changelist.tests.ChangeListTests.test_clear_all_filters_link", # noqa + "admin_changelist.tests.ChangeListTests.test_clear_all_filters_link_callable_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_no_clear_all_filters_link", # noqa + "admin_changelist.tests.ChangeListTests.test_no_duplicates_for_inherited_m2m_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_no_duplicates_for_m2m_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_no_duplicates_for_m2m_to_inherited_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_no_duplicates_for_many_to_many_at_second_level_in_search_fields", # noqa + "admin_changelist.tests.ChangeListTests.test_no_duplicates_for_non_unique_related_object_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_no_duplicates_for_non_unique_related_object_in_search_fields", # noqa + "admin_changelist.tests.ChangeListTests.test_no_duplicates_for_through_m2m_at_second_level_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_no_duplicates_for_through_m2m_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_no_exists_for_m2m_in_list_filter_without_params", # noqa + "admin_changelist.tests.ChangeListTests.test_total_ordering_optimization_meta_constraints", # noqa + "admin_docs.test_middleware.XViewMiddlewareTest.test_no_auth_middleware", # noqa + "admin_docs.test_views.AdminDocViewDefaultEngineOnly.test_template_detail_path_traversal", # noqa + "admin_inlines.tests.TestInline.test_custom_form_tabular_inline_extra_field_label", # noqa + "admin_inlines.tests.TestInline.test_inlines_singular_heading_one_to_one", # noqa + "admin_inlines.tests.TestInline.test_non_editable_custom_form_tabular_inline_extra_field_label", # noqa + "admin_views.test_multidb.MultiDatabaseTests.test_delete_view", # noqa + "auth_tests.test_auth_backends.AuthenticateTests.test_authenticate_sensitive_variables", # noqa + "auth_tests.test_auth_backends.AuthenticateTests.test_clean_credentials_sensitive_variables", # noqa + "auth_tests.test_auth_backends.AuthenticateTests.test_skips_backends_with_decorated_method", # noqa + "auth_tests.test_auth_backends.BaseBackendTest.test_get_all_permissions", # noqa + "auth_tests.test_auth_backends.BaseBackendTest.test_get_group_permissions", # noqa + "auth_tests.test_auth_backends.BaseBackendTest.test_get_user_permissions", # noqa + "auth_tests.test_auth_backends.BaseBackendTest.test_has_perm", # noqa + "auth_tests.test_auth_backends.CustomPermissionsUserModelBackendTest.test_authentication_without_credentials", # noqa + "auth_tests.test_auth_backends.ExtensionUserModelBackendTest.test_authentication_without_credentials", # noqa + "auth_tests.test_auth_backends.ModelBackendTest.test_authentication_without_credentials", # noqa + "auth_tests.test_basic.BasicTestCase.test_superuser_no_email_or_password", # noqa + "auth_tests.test_basic.BasicTestCase.test_superuser_no_email_or_password", # noqa + "auth_tests.test_basic.BasicTestCase.test_superuser_no_email_or_password", # noqa + "auth_tests.test_basic.BasicTestCase.test_superuser_no_email_or_password", # noqa + "auth_tests.test_decorators.LoginRequiredTestCase.test_callable", # noqa + "auth_tests.test_decorators.LoginRequiredTestCase.test_login_required", # noqa + "auth_tests.test_decorators.LoginRequiredTestCase.test_login_required_next_url", # noqa + "auth_tests.test_decorators.LoginRequiredTestCase.test_view", # noqa + "auth_tests.test_forms.AdminPasswordChangeFormTest.test_html_autocomplete_attributes", # noqa + "auth_tests.test_forms.AuthenticationFormTest.test_html_autocomplete_attributes", # noqa + "auth_tests.test_forms.AuthenticationFormTest.test_username_field_autocapitalize_none", # noqa + "auth_tests.test_forms.PasswordChangeFormTest.test_html_autocomplete_attributes", # noqa + "auth_tests.test_forms.PasswordResetFormTest.test_html_autocomplete_attributes", # noqa + "auth_tests.test_forms.SetPasswordFormTest.test_html_autocomplete_attributes", # noqa + "auth_tests.test_forms.UserChangeFormTest.test_username_field_autocapitalize_none", # noqa + "auth_tests.test_forms.UserCreationFormTest.test_html_autocomplete_attributes", # noqa + "auth_tests.test_forms.UserCreationFormTest.test_username_field_autocapitalize_none", # noqa + "auth_tests.test_management.CreatesuperuserManagementCommandTestCase.test_environment_variable_non_interactive", # noqa + "auth_tests.test_management.CreatesuperuserManagementCommandTestCase.test_fields_with_m2m", # noqa + "auth_tests.test_management.CreatesuperuserManagementCommandTestCase.test_fields_with_m2m_interactive", # noqa + "auth_tests.test_management.CreatesuperuserManagementCommandTestCase.test_fields_with_m2m_interactive_blank", # noqa + "auth_tests.test_management.CreatesuperuserManagementCommandTestCase.test_ignore_environment_variable_interactive", # noqa + "auth_tests.test_management.CreatesuperuserManagementCommandTestCase.test_ignore_environment_variable_non_interactive", # noqa + "auth_tests.test_management.GetDefaultUsernameTestCase.test_with_database", # noqa + "auth_tests.test_management.MultiDBCreatesuperuserTestCase.test_createsuperuser_command_suggested_username_with_database_option", # noqa + "auth_tests.test_middleware.TestAuthenticationMiddleware.test_no_password_change_does_not_invalidate_legacy_session", # noqa + "auth_tests.test_middleware.TestAuthenticationMiddleware.test_no_session", # noqa + "auth_tests.test_middleware.TestAuthenticationMiddleware.test_session_default_hashing_algorithm", # noqa + "auth_tests.test_models.UserManagerTestCase.test_runpython_manager_methods", # noqa + "auth_tests.test_models.UserWithPermTestCase.test_backend_without_with_perm", # noqa + "auth_tests.test_models.UserWithPermTestCase.test_basic", # noqa + "auth_tests.test_models.UserWithPermTestCase.test_custom_backend", # noqa + "auth_tests.test_models.UserWithPermTestCase.test_custom_backend_pass_obj", # noqa + "auth_tests.test_models.UserWithPermTestCase.test_invalid_backend_type", # noqa + "auth_tests.test_models.UserWithPermTestCase.test_invalid_permission_name", # noqa + "auth_tests.test_models.UserWithPermTestCase.test_invalid_permission_type", # noqa + "auth_tests.test_models.UserWithPermTestCase.test_multiple_backends", # noqa + "auth_tests.test_models.UserWithPermTestCase.test_nonexistent_backend", # noqa + "auth_tests.test_models.UserWithPermTestCase.test_nonexistent_permission", # noqa + "auth_tests.test_password_reset_timeout_days.DeprecationTests.test_timeout", # noqa + "auth_tests.test_remote_user.AllowAllUsersRemoteUserBackendTest.test_csrf_validation_passes_after_process_request_login", # noqa + "auth_tests.test_remote_user.CustomHeaderRemoteUserTest.test_csrf_validation_passes_after_process_request_login", # noqa + "auth_tests.test_remote_user.PersistentRemoteUserTest.test_csrf_validation_passes_after_process_request_login", # noqa + "auth_tests.test_remote_user.RemoteUserCustomTest.test_csrf_validation_passes_after_process_request_login", # noqa + "auth_tests.test_remote_user.RemoteUserTest.test_csrf_validation_passes_after_process_request_login", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_password_change_done_view", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_password_reset_change_view", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_password_reset_complete_view", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_password_reset_confirm_view_custom_username_hint", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_password_reset_confirm_view_invalid_token", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_password_reset_confirm_view_valid_token", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_password_reset_done_view", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_password_reset_view", # noqa + "auth_tests.test_tokens.TokenGeneratorTest.test_legacy_days_timeout", # noqa + "auth_tests.test_tokens.TokenGeneratorTest.test_legacy_token_validation", # noqa + "auth_tests.test_tokens.TokenGeneratorTest.test_token_default_hashing_algorithm", # noqa + "auth_tests.test_tokens.TokenGeneratorTest.test_token_with_different_email", # noqa + "auth_tests.test_tokens.TokenGeneratorTest.test_token_with_different_email", # noqa + "auth_tests.test_tokens.TokenGeneratorTest.test_token_with_different_email", # noqa + "auth_tests.test_views.LoginTest.test_legacy_session_key_flushed_on_login", # noqa + "auth_tests.test_views.PasswordResetTest.test_confirm_custom_reset_url_token", # noqa + "auth_tests.test_views.PasswordResetTest.test_confirm_custom_reset_url_token_link_redirects_to_set_password_page", # noqa + "datetimes.tests.DateTimesTests.test_datetimes_ambiguous_and_invalid_times", # noqa + "db_functions.comparison.test_cast.CastTests.test_cast_to_duration", # noqa + "fixtures.tests.TestCaseFixtureLoadingTests.test_class_fixtures", # noqa + "generic_inline_admin.tests.GenericInlineAdminParametersTest.test_max_num_param", # noqa + "queries.tests.Queries1Tests.test_excluded_intermediary_m2m_table_joined", # noqa + "queries.tests.Queries1Tests.test_field_with_filterable", # noqa + "queries.tests.Queries1Tests.test_negate_field", # noqa + "queries.tests.Queries1Tests.test_order_by_raw_column_alias_warning", # noqa + "queries.tests.Queries1Tests.test_order_by_rawsql", # noqa + "queries.tests.Queries4Tests.test_combine_or_filter_reuse", # noqa + "queries.tests.Queries4Tests.test_filter_reverse_non_integer_pk", # noqa + "schema.tests.SchemaTests.test_alter_field_default_doesnt_perform_queries", # noqa + "sitemaps_tests.test_http.HTTPSitemapTests.test_alternate_i18n_sitemap_index", # noqa + "sitemaps_tests.test_http.HTTPSitemapTests.test_alternate_i18n_sitemap_limited", # noqa + "sitemaps_tests.test_http.HTTPSitemapTests.test_alternate_i18n_sitemap_xdefault", # noqa + "sitemaps_tests.test_http.HTTPSitemapTests.test_simple_i18n_sitemap_index", # noqa + "test_client.tests.ClientTest.test_exc_info", # noqa + "test_client.tests.ClientTest.test_exc_info_none", # noqa + "test_client.tests.ClientTest.test_follow_307_and_308_get_head_query_string", # noqa + "test_client.tests.ClientTest.test_follow_307_and_308_preserves_query_string", # noqa + ) + else: + skip_tests += ( + # Untyped parameters are not supported: + # https://github.com/GoogleCloudPlatform/cloud-spanner-emulator#features-and-limitations + "queries.tests.Queries1Tests.test_ticket9411", # noqa + "admin_changelist.tests.ChangeListTests.test_distinct_for_inherited_m2m_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_distinct_for_m2m_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_distinct_for_m2m_to_inherited_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_distinct_for_many_to_many_at_second_level_in_search_fields", # noqa + "admin_changelist.tests.ChangeListTests.test_distinct_for_non_unique_related_object_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_distinct_for_non_unique_related_object_in_search_fields", # noqa + "admin_changelist.tests.ChangeListTests.test_distinct_for_through_m2m_at_second_level_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_distinct_for_through_m2m_in_list_filter", # noqa + "admin_changelist.tests.ChangeListTests.test_no_distinct_for_m2m_in_list_filter_without_params", # noqa + "aggregation.tests.AggregateTestCase.test_missing_output_field_raises_error", # noqa + "auth_tests.test_decorators.LoginRequiredTestCase.testCallable", # noqa + "auth_tests.test_decorators.LoginRequiredTestCase.testLoginRequired", # noqa + "auth_tests.test_decorators.LoginRequiredTestCase.testLoginRequiredNextUrl", # noqa + "auth_tests.test_decorators.LoginRequiredTestCase.testView", # noqa + "auth_tests.test_remote_user_deprecation.RemoteUserCustomTest.test_configure_user_deprecation_warning", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_PasswordChangeDoneView", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetChangeView", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetCompleteView", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetConfirmView_invalid_token", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetConfirmView_valid_token", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetDoneView", # noqa + "auth_tests.test_templates.AuthTemplateTests.test_PasswordResetView", # noqa + "fixtures.tests.TestCaseFixtureLoadingTests.testClassFixtures", # noqa + "fixtures_model_package.tests.SampleTestCase.testClassFixtures", # noqa + "generic_inline_admin.tests.GenericInlineAdminParametersTest.testMaxNumParam", # noqa + "migrations.test_operations.OperationTests.test_autofield_foreignfield_growth", # noqa + "ordering.tests.OrderingTests.test_deprecated_values_annotate", # noqa + "queries.tests.Queries1Tests.test_ticket2902", # noqa + "schema.tests.SchemaTests.test_alter_field_default_doesnt_perfom_queries", # noqa + "sitemaps_tests.test_http.HTTPSitemapTests.test_simple_i18nsitemap_index", # noqa + ) diff --git a/django_spanner/introspection.py b/django_spanner/introspection.py index b95ea3e629..938718d2d5 100644 --- a/django_spanner/introspection.py +++ b/django_spanner/introspection.py @@ -11,6 +11,8 @@ ) from django.db.models import Index from google.cloud.spanner_v1 import TypeCode +from django_spanner import USE_EMULATOR +from django_spanner import USING_DJANGO_3 class DatabaseIntrospection(BaseDatabaseIntrospection): @@ -25,7 +27,28 @@ class DatabaseIntrospection(BaseDatabaseIntrospection): TypeCode.STRING: "CharField", TypeCode.TIMESTAMP: "DateTimeField", TypeCode.NUMERIC: "DecimalField", + TypeCode.JSON: "JSONField", } + if USE_EMULATOR: + # Emulator does not support table_type yet. + # https://github.com/GoogleCloudPlatform/cloud-spanner-emulator/issues/43 + LIST_TABLE_SQL = """ + SELECT + t.table_name, t.table_name + FROM + information_schema.tables AS t + WHERE + t.table_catalog = '' and t.table_schema = '' + """ + else: + LIST_TABLE_SQL = """ + SELECT + t.table_name, t.table_type + FROM + information_schema.tables AS t + WHERE + t.table_catalog = '' and t.table_schema = '' + """ def get_field_type(self, data_type, description): """A hook for a Spanner database to use the cursor description to @@ -53,8 +76,15 @@ def get_table_list(self, cursor): :rtype: list :returns: A list of table and view names in the current database. """ + results = cursor.run_sql_in_snapshot(self.LIST_TABLE_SQL) + tables = [] # The second TableInfo field is 't' for table or 'v' for view. - return [TableInfo(row[0], "t") for row in cursor.list_tables()] + for row in results: + table_type = "t" + if row[1] == "VIEW": + table_type = "v" + tables.append(TableInfo(row[0], table_type)) + return tables def get_table_description(self, cursor, table_name): """Return a description of the table with the DB-API cursor.description @@ -86,18 +116,33 @@ def get_table_description(self, cursor, table_name): internal_size = int(internal_size) else: internal_size = None - descriptions.append( - FieldInfo( - column_name, - type_code, - None, # display_size - internal_size, - None, # precision - None, # scale - details.null_ok, - None, # default + if USING_DJANGO_3: + descriptions.append( + FieldInfo( + column_name, + type_code, + None, # display_size + internal_size, + None, # precision + None, # scale + details.null_ok, + None, # default + None, # collation + ) + ) + else: + descriptions.append( + FieldInfo( + column_name, + type_code, + None, # display_size + internal_size, + None, # precision + None, # scale + details.null_ok, + None, # default + ) ) - ) return descriptions diff --git a/django_spanner/lookups.py b/django_spanner/lookups.py index d9a54982f2..8f837733c5 100644 --- a/django_spanner/lookups.py +++ b/django_spanner/lookups.py @@ -100,10 +100,18 @@ def iexact(self, compiler, connection): # lhs_sql is the expression/column to use as the regular expression. # Use concat to make the value case-insensitive. lhs_sql = "CONCAT('^(?i)', " + lhs_sql + ", '$')" - if not self.rhs_is_direct_value() and not params: - # If rhs is not a direct value and parameter is not present we want - # to have only 1 formatable argument in rhs_sql else we need 2. - rhs_sql = rhs_sql.replace("%%s", "%s") + if not self.rhs_is_direct_value(): + # If rhs is not a direct value + if not params: + # if params is not present, then we have only 1 formatable + # argument in rhs_sql. + rhs_sql = rhs_sql.replace("%%s", "%s") + else: + # If params is present and rhs_sql is to be replaced as well. + # Example: model_fields.test_uuid.TestQuerying.test_iexact. + rhs_sql = rhs_sql.replace("%%s", "__PLACEHOLDER_FOR_LHS_SQL__") + rhs_sql = rhs_sql.replace("%s", "%%s") + rhs_sql = rhs_sql.replace("__PLACEHOLDER_FOR_LHS_SQL__", "%s") # rhs_sql is REGEXP_CONTAINS(%s, %%s), and lhs_sql is the column name. return rhs_sql % lhs_sql, params diff --git a/django_spanner/operations.py b/django_spanner/operations.py index 48a3e3cef3..5cc78b161c 100644 --- a/django_spanner/operations.py +++ b/django_spanner/operations.py @@ -377,7 +377,7 @@ def datetime_extract_sql(self, lookup_type, field_name, tzname): :rtype: str :returns: A SQL statement for extracting. """ - tzname = tzname if settings.USE_TZ else "UTC" + tzname = tzname if settings.USE_TZ and tzname else "UTC" lookup_type = self.extract_names.get(lookup_type, lookup_type) return 'EXTRACT(%s FROM %s AT TIME ZONE "%s")' % ( lookup_type, @@ -403,7 +403,7 @@ def time_extract_sql(self, lookup_type, field_name): field_name, ) - def date_trunc_sql(self, lookup_type, field_name): + def date_trunc_sql(self, lookup_type, field_name, tzname=None): """Truncate date in the lookup. :type lookup_type: str @@ -412,6 +412,10 @@ def date_trunc_sql(self, lookup_type, field_name): :type field_name: str :param field_name: The name of the field. + :type tzname: str + :param tzname: The name of the timezone. This is ignored because + Spanner does not support Timezone conversion in DATE_TRUNC function. + :rtype: str :returns: A SQL statement for truncating. """ @@ -429,7 +433,7 @@ def date_trunc_sql(self, lookup_type, field_name): sql = "DATE_ADD(CAST(" + sql + " AS DATE), INTERVAL 1 DAY)" return sql - def datetime_trunc_sql(self, lookup_type, field_name, tzname): + def datetime_trunc_sql(self, lookup_type, field_name, tzname="UTC"): """Truncate datetime in the lookup. :type lookup_type: str @@ -438,11 +442,14 @@ def datetime_trunc_sql(self, lookup_type, field_name, tzname): :type field_name: str :param field_name: The name of the field. + :type tzname: str + :param tzname: The name of the timezone. + :rtype: str :returns: A SQL statement for truncating. """ # https://cloud.google.com/spanner/docs/functions-and-operators#timestamp_trunc - tzname = tzname if settings.USE_TZ else "UTC" + tzname = tzname if settings.USE_TZ and tzname else "UTC" if lookup_type == "week": # Spanner truncates to Sunday but Django expects Monday. First, # subtract a day so that a Sunday will be truncated to the previous @@ -458,7 +465,7 @@ def datetime_trunc_sql(self, lookup_type, field_name, tzname): sql = "TIMESTAMP_ADD(" + sql + ", INTERVAL 1 DAY)" return sql - def time_trunc_sql(self, lookup_type, field_name): + def time_trunc_sql(self, lookup_type, field_name, tzname="UTC"): """Truncate time in the lookup. :type lookup_type: str @@ -467,11 +474,19 @@ def time_trunc_sql(self, lookup_type, field_name): :type field_name: str :param field_name: The name of the field. + :type tzname: str + :param tzname: The name of the timezone. Defaults to 'UTC' For backward compatability. + :rtype: str :returns: A SQL statement for truncating. """ # https://cloud.google.com/spanner/docs/functions-and-operators#timestamp_trunc - return 'TIMESTAMP_TRUNC(%s, %s, "UTC")' % (field_name, lookup_type) + tzname = tzname if settings.USE_TZ and tzname else "UTC" + return 'TIMESTAMP_TRUNC(%s, %s, "%s")' % ( + field_name, + lookup_type, + tzname, + ) def datetime_cast_date_sql(self, field_name, tzname): """Cast date in the lookup. @@ -487,7 +502,7 @@ def datetime_cast_date_sql(self, field_name, tzname): :returns: A SQL statement for casting. """ # https://cloud.google.com/spanner/docs/functions-and-operators#date - tzname = tzname if settings.USE_TZ else "UTC" + tzname = tzname if settings.USE_TZ and tzname else "UTC" return 'DATE(%s, "%s")' % (field_name, tzname) def datetime_cast_time_sql(self, field_name, tzname): @@ -503,7 +518,7 @@ def datetime_cast_time_sql(self, field_name, tzname): :rtype: str :returns: A SQL statement for casting. """ - tzname = tzname if settings.USE_TZ else "UTC" + tzname = tzname if settings.USE_TZ and tzname else "UTC" # Cloud Spanner doesn't have a function for converting # TIMESTAMP to another time zone. return ( @@ -549,6 +564,10 @@ def combine_expression(self, connector, sub_expressions): return "MOD(%s)" % ", ".join(sub_expressions) elif connector == "^": return "POWER(%s)" % ", ".join(sub_expressions) + elif connector == "#": + # Connector '#' represents Bit Xor in django. + # Spanner represents the same fuction with '^' symbol. + return super().combine_expression("^", sub_expressions) elif connector == ">>": lhs, rhs = sub_expressions # Use an alternate computation because Cloud Sapnner's '>>' diff --git a/django_spanner/schema.py b/django_spanner/schema.py index 247358857a..4f40f0574d 100644 --- a/django_spanner/schema.py +++ b/django_spanner/schema.py @@ -4,10 +4,11 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +import uuid from django.db import NotSupportedError from django.db.backends.base.schema import BaseDatabaseSchemaEditor from django_spanner._opentelemetry_tracing import trace_call -from django_spanner import USE_EMULATOR +from django_spanner import USE_EMULATOR, USING_DJANGO_3 class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): @@ -60,7 +61,15 @@ def create_model(self, model): # Check constraints can go on the column SQL here db_params = field.db_parameters(connection=self.connection) if db_params["check"]: - definition += " " + self.sql_check_constraint % db_params + definition += ( + ", CONSTRAINT constraint_%s_%s_%s " + % ( + model._meta.db_table, + self.quote_name(field.name), + uuid.uuid4().hex[:6].lower(), + ) + + self.sql_check_constraint % db_params + ) # Autoincrement SQL (for backends with inline variant) col_type_suffix = field.db_type_suffix(connection=self.connection) if col_type_suffix: @@ -124,6 +133,7 @@ def create_model(self, model): trace_attributes = { "model_name": self.quote_name(model._meta.db_table) } + with trace_call( "CloudSpannerDjango.create_model", self.connection, @@ -206,7 +216,15 @@ def add_field(self, model, field): # Check constraints can go on the column SQL here db_params = field.db_parameters(connection=self.connection) if db_params["check"]: - definition += " " + self.sql_check_constraint % db_params + definition += ( + ", CONSTRAINT constraint_%s_%s_%s " + % ( + model._meta.db_table, + self.quote_name(field.name), + uuid.uuid4().hex[:6].lower(), + ) + + self.sql_check_constraint % db_params + ) # Build the SQL and run it sql = self.sql_create_column % { "table": self.quote_name(model._meta.db_table), @@ -378,6 +396,8 @@ def add_index(self, model, index): def quote_value(self, value): # A more complete implementation isn't currently required. + if isinstance(value, str): + return "'%s'" % value.replace("'", "''") return str(value) def _alter_field( @@ -450,7 +470,7 @@ def _alter_field( self.connection, trace_attributes, ): - self.execute(self._create_index_sql(model, [new_field])) + self.execute(self._create_index_sql(model, fields=[new_field])) def _alter_column_type_sql(self, model, old_field, new_field, new_type): # Spanner needs to use sql_alter_column_not_null if the field is @@ -481,11 +501,30 @@ def _check_sql(self, name, check): "constraint": self.sql_check_constraint % {"check": check}, } - def _unique_sql(self, model, fields, name, condition=None): + def _unique_sql( + self, + model, + fields, + name, + condition=None, + deferrable=None, # Spanner does not require this parameter + include=None, + opclasses=None, + ): # Inline constraints aren't supported, so create the index separately. - sql = self._create_unique_sql( - model, fields, name=name, condition=condition - ) + if USING_DJANGO_3: + sql = self._create_unique_sql( + model, + fields, + name=name, + condition=condition, + include=include, + opclasses=opclasses, + ) + else: + sql = self._create_unique_sql( + model, fields, name=name, condition=condition + ) if sql: self.deferred_sql.append(sql) return None diff --git a/django_spanner/utils.py b/django_spanner/utils.py index 6fb40db812..25b06350ca 100644 --- a/django_spanner/utils.py +++ b/django_spanner/utils.py @@ -7,18 +7,17 @@ import django import sqlparse from django.core.exceptions import ImproperlyConfigured -from django.utils.version import get_version_tuple -def check_django_compatability(): +def check_django_compatability(supported_django_versions): """ Verify that this version of django-spanner is compatible with the installed - version of Django. For example, any django-spanner 2.2.x is compatible - with Django 2.2.y. + version of Django. For example, django-spanner is compatible + with Django 2.2.y and 3.2.z """ from . import __version__ - if django.VERSION[:2] != get_version_tuple(__version__)[:2]: + if django.VERSION[:2] not in supported_django_versions: raise ImproperlyConfigured( "You must use the latest version of django-spanner {A}.{B}.x " "with Django {A}.{B}.y (found django-spanner {C}).".format( diff --git a/django_test_suite.sh b/django_test_suite.sh index 17173cc2f9..c00e833859 100755 --- a/django_test_suite.sh +++ b/django_test_suite.sh @@ -60,6 +60,7 @@ SECRET_KEY = 'spanner_tests_secret_key' PASSWORD_HASHERS = [ 'django.contrib.auth.hashers.MD5PasswordHasher', ] +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' ! } diff --git a/django_test_suite_3.2.sh b/django_test_suite_3.2.sh new file mode 100755 index 0000000000..c99b0032f2 --- /dev/null +++ b/django_test_suite_3.2.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +# Copyright (c) 2020 Google LLC. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +set -x pipefail + +sudo apt-get update -y +sudo apt-get install -y libmemcached-dev + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +export DJANGO_TESTS_DIR="django_tests_dir" +mkdir -p $DJANGO_TESTS_DIR + +if [ $SPANNER_EMULATOR_HOST != 0 ] +then + pip3 install . + git clone --depth 1 --single-branch --branch "stable/spanner/3.2.x" https://github.com/vi3k6i5/django.git $DJANGO_TESTS_DIR/django3.2 +fi + +# Install dependencies for Django tests. +sudo apt-get update +sudo apt-get install -y libffi-dev libjpeg-dev zlib1g-devel + +cd $DJANGO_TESTS_DIR/django3.2 && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../ + +python3 create_test_instance.py + +# If no SPANNER_TEST_DB is set, generate a unique one +# so that we can have multiple tests running without +# conflicting which changes and constraints. We'll always +# cleanup the created database. +TEST_DBNAME=${SPANNER_TEST_DB:-$(python3 -c 'import os, time; print(chr(ord("a") + time.time_ns() % 26)+os.urandom(10).hex())')} +TEST_DBNAME_OTHER="$TEST_DBNAME-ot" +INSTANCE=${SPANNER_TEST_INSTANCE:-django-tests} +PROJECT=${PROJECT_ID} +SETTINGS_FILE="$TEST_DBNAME-settings" +TESTS_DIR=${DJANGO_TESTS_DIR:-django_tests} + +create_settings() { + cat << ! > "$SETTINGS_FILE.py" +DATABASES = { + 'default': { + 'ENGINE': 'django_spanner', + 'PROJECT': "$PROJECT", + 'INSTANCE': "$INSTANCE", + 'NAME': "$TEST_DBNAME", + }, + 'other': { + 'ENGINE': 'django_spanner', + 'PROJECT': "$PROJECT", + 'INSTANCE': "$INSTANCE", + 'NAME': "$TEST_DBNAME_OTHER", + }, +} +SECRET_KEY = 'spanner_tests_secret_key' +PASSWORD_HASHERS = [ + 'django.contrib.auth.hashers.MD5PasswordHasher', +] +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +! +} + +cd $TESTS_DIR/django3.2/tests +create_settings + +EXIT_STATUS=0 +for DJANGO_TEST_APP in $DJANGO_TEST_APPS +do + python3 runtests.py $DJANGO_TEST_APP --verbosity=3 --noinput --settings $SETTINGS_FILE || EXIT_STATUS=$? +done +exit $EXIT_STATUS diff --git a/docs/index.rst b/docs/index.rst index ec9b23a055..950bb65f28 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,3 +25,11 @@ For a list of all ``google-cloud-spanner-django`` releases: :maxdepth: 2 changelog + +Limitations +--------- + +.. toctree:: + :maxdepth: 1 + + limitations \ No newline at end of file diff --git a/docs/limitations.rst b/docs/limitations.rst new file mode 100644 index 0000000000..2909743922 --- /dev/null +++ b/docs/limitations.rst @@ -0,0 +1,165 @@ +Current limitations +------------------- + +``AutoField`` generates random IDs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spanner doesn't have support for auto-generating primary key values. +Therefore, ``django-google-spanner`` monkey-patches ``AutoField`` to generate a +random UUID4. It generates a default using ``Field``'s ``default`` option which +means ``AutoField``\ s will have a value when a model instance is created. For +example: + +:: + + >>> ExampleModel() + >>> ExampleModel.pk + 4229421414948291880 + +To avoid +`hotspotting `__, +these IDs are not monotonically increasing. This means that sorting +models by ID isn't guaranteed to return them in the order in which they +were created. + +``ForeignKey`` constraints aren't created (`#313 `__) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spanner does not support ``ON DELETE CASCADE`` when creating foreign-key +constraints, so this is not supported in ``django-google-spanner``. + + +No native support for ``DecimalField`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spanner's support for `Decimal `__ +types is limited to +`NUMERIC `__ +precision. Higher-precision values can be stored as strings instead. + + +``Meta.order_with_respect_to`` model option isn't supported +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This feature uses a column name that starts with an underscore +(``_order``) which Spanner doesn't allow. + +Random ``QuerySet`` ordering isn't supported +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spanner does not support it and will throw an exception. For example: + +:: + + >>> ExampleModel.objects.order_by('?') + ... + django.db.utils.ProgrammingError: 400 Function not found: RANDOM ... FROM + example_model ORDER BY RANDOM() ASC + +Schema migrations +~~~~~~~~~~~~~~~~~ + +There are some limitations on schema changes to consider: + +- No support for renaming tables and columns; +- A column's type can't be changed; +- A table's primary key can't be altered. + +``DurationField`` arithmetic doesn't work with ``DateField`` values (`#253 `__) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spanner requires using different functions for arithmetic depending on +the column type: + +- ``TIMESTAMP`` columns (``DateTimeField``) require ``TIMESTAMP_ADD`` + or ``TIMESTAMP_SUB`` +- ``DATE`` columns (``DateField``) require ``DATE_ADD`` or ``DATE_SUB`` + +Django does not provide ways to determine which database function to +use. ``DatabaseOperations.combine_duration_expression()`` arbitrarily uses +``TIMESTAMP_ADD`` and ``TIMESTAMP_SUB``. Therefore, if you use a +``DateField`` in a ``DurationField`` expression, you'll likely see an error +such as: + +:: + + "No matching signature for function TIMESTAMP\_ADD for argument types: + DATE, INTERVAL INT64 DATE\_TIME\_PART." + +Computations that yield FLOAT64 values cannot be assigned to INT64 columns +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spanner does not support this (`#331 +`__) and will +throw an error: + +:: + + >>> ExampleModel.objects.update(integer=F('integer') / 2) + ... + django.db.utils.ProgrammingError: 400 Value of type FLOAT64 cannot be + assigned to integer, which has type INT64 [at 1:46]\nUPDATE + example_model SET integer = (example_model.integer /... + +Addition with null values crash +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Additions cannot include ``None`` values. For example: + +:: + + >>> Book.objects.annotate(adjusted_rating=F('rating') + None) + ... + google.api_core.exceptions.InvalidArgument: 400 Operands of + cannot be literal + NULL ... + +stddev() and variance() function call with sample population only +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spanner supports `stddev()` and `variance()` functions (`link `__). + +Django’s Variance and StdDev database functions have 2 modes. +One with full population `STDDEV_POP` and another with sample population `STDDEV_SAMP` and `VAR_SAMP`. +Currently spanner only supports these functions with samples and not the full population `STDDEV_POP`, + + +Interleaving is not supported currently +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Interleaving is a feature that is supported by spanner database `link `_. +But currently django spanner does not support this feature, more details on this is discussed in this `github issue `_. + +Update object by passing primary key +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +In django3.1 a new feature was introduced, `._state.adding`, +this allowed spanner to resolve `this bug `_. + +But introduced a new issue with spanner django. Calling `instance.save()` an object after setting it's primary key to an existing primary key value, +will cause a `IntegrityError` as follows: `django.db.utils.IntegrityError: (1062, "Duplicate entry ....` + +The workaround for this is to update `._state.adding` to `False`. +Example: +.. code:: python + + >>> # This test case passes. + >>> def test_update_primary_with_default(self): + >>> obj = PrimaryKeyWithDefault() + >>> obj.save() + >>> obj_2 = PrimaryKeyWithDefault(uuid=obj.uuid) + >>> obj_2._state.adding = False + >>> obj_2.save() + + >>> # This test case fails with `IntegrityError`. + >>> def test_update_primary_with_default(self): + >>> obj = PrimaryKeyWithDefault() + >>> obj.save() + >>> obj_2 = PrimaryKeyWithDefault(uuid=obj.uuid) + >>> obj_2.save() + +More details about this issue can be tracked `here `_. + +Support for Json field is currently not there +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This library currently does not support json. For more info on json support support follow this `github issue `_. + diff --git a/noxfile.py b/noxfile.py index 3b51d73841..0f295b8594 100644 --- a/noxfile.py +++ b/noxfile.py @@ -66,17 +66,17 @@ def lint_setup_py(session): ) -def default(session): +def default(session, django_version="2.2"): # Install all test dependencies, then install this package in-place. session.install( - "django~=2.2", + "django~={}".format(django_version), "mock", "mock-import", "pytest", "pytest-cov", "coverage", "sqlparse==0.3.0", - "google-cloud-spanner==3.0.0", + "google-cloud-spanner==3.11.1", "opentelemetry-api==1.1.0", "opentelemetry-sdk==1.1.0", "opentelemetry-instrumentation==0.20b0", @@ -101,11 +101,13 @@ def default(session): @nox.session(python=UNIT_TEST_PYTHON_VERSIONS) def unit(session): """Run the unit test suite.""" + print("Unit tests with django 2.2") default(session) + print("Unit tests with django 3.2") + default(session, django_version="3.2") -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) -def system(session): +def system_test(session, django_version="2.2"): """Run the system test suite.""" constraints_path = str( CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" @@ -136,7 +138,7 @@ def system(session): # Install all test dependencies, then install this package into the # virtualenv's dist-packages. session.install( - "django~=2.2", + "django~={}".format(django_version), "mock", "pytest", "google-cloud-testutils", @@ -154,6 +156,14 @@ def system(session): ) +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def system(session): + print("System tests with django 2.2") + system_test(session) + print("System tests with django 3.2") + system_test(session, django_version="3.2") + + @nox.session(python=DEFAULT_PYTHON_VERSION) def cover(session): """Run the final coverage report. diff --git a/setup.py b/setup.py index a3143566bb..c089f4e71a 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ # 'Development Status :: 4 - Beta' # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 4 - Beta" -dependencies = ["sqlparse >= 0.3.0", "google-cloud-spanner >= 3.0.0"] +dependencies = ["sqlparse >= 0.3.0", "google-cloud-spanner >= 3.11.1"] extras = { "tracing": [ "opentelemetry-api >= 1.1.0", @@ -63,9 +63,11 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Utilities", "Framework :: Django", "Framework :: Django :: 2.2", + "Framework :: Django :: 3.2", ], extras_require=extras, python_requires=">=3.6", diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 7573802344..8bb73e0b1b 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -6,7 +6,7 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 sqlparse==0.3.0 -google-cloud-spanner==3.0.0 +google-cloud-spanner==3.11.1 opentelemetry-api==1.1.0 opentelemetry-sdk==1.1.0 opentelemetry-instrumentation==0.20b0 diff --git a/tests/system/django_spanner/models.py b/tests/system/django_spanner/models.py index f7153ba994..cb75aba9e7 100644 --- a/tests/system/django_spanner/models.py +++ b/tests/system/django_spanner/models.py @@ -8,6 +8,7 @@ Different models used by system tests in django-spanner code. """ from django.db import models +from django_spanner import USING_DJANGO_3 class Author(models.Model): @@ -34,3 +35,9 @@ class Meta: name="check_start_date", ), ] + + +if USING_DJANGO_3: + + class Detail(models.Model): + value = models.JSONField() diff --git a/tests/system/django_spanner/test_json_field.py b/tests/system/django_spanner/test_json_field.py new file mode 100644 index 0000000000..d3cbc97dd4 --- /dev/null +++ b/tests/system/django_spanner/test_json_field.py @@ -0,0 +1,51 @@ +# Copyright 2021 Google LLC +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +import unittest +from django.test import TransactionTestCase +from django.db import connection +from django_spanner import USE_EMULATOR +from django_spanner import USING_DJANGO_3 +from tests.system.django_spanner.utils import ( + setup_instance, + teardown_instance, + setup_database, + teardown_database, +) + +if USING_DJANGO_3: + from .models import Detail + + +@unittest.skipIf(USE_EMULATOR, "Jsonfield is not implemented in emulator.") +class TestJsonField(TransactionTestCase): + @classmethod + def setUpClass(cls): + setup_instance() + setup_database() + with connection.schema_editor() as editor: + # Create the tables + editor.create_model(Detail) + + @classmethod + def tearDownClass(cls): + with connection.schema_editor() as editor: + # delete the table + editor.delete_model(Detail) + teardown_database() + teardown_instance() + + def test_insert_and_fetch_value(self): + """ + Tests model object creation with Detail model. + Inserting json data into the model and retrieving it. + """ + json_data = Detail(value={"name": "Jakob", "age": "26"}) + json_data.save() + qs1 = Detail.objects.all() + self.assertEqual(qs1[0].value, {"name": "Jakob", "age": "26"}) + # Delete data from Detail table. + Detail.objects.all().delete() diff --git a/tests/unit/django_spanner/test_functions.py b/tests/unit/django_spanner/test_functions.py index 00b431b73b..3624aac1c5 100644 --- a/tests/unit/django_spanner/test_functions.py +++ b/tests/unit/django_spanner/test_functions.py @@ -6,6 +6,7 @@ from tests.unit.django_spanner.simple_test import SpannerSimpleTestClass from django_spanner.compiler import SQLCompiler +from django_spanner import USING_DJANGO_3 from django.db.models import CharField, FloatField, Value from django.db.models.functions import ( Cast, @@ -176,10 +177,18 @@ def test_pi(self): compiler = SQLCompiler(q1.query, self.connection, "default") sql_query, params = compiler.query.as_sql(compiler, self.connection) + if USING_DJANGO_3: + expected_sql = ( + "SELECT tests_author.num FROM tests_author WHERE tests_author.num " + + "= 3.141592653589793" + ) + else: + expected_sql = ( + "SELECT tests_author.num FROM tests_author WHERE tests_author.num " + + "= (3.141592653589793)" + ) self.assertEqual( - sql_query, - "SELECT tests_author.num FROM tests_author WHERE tests_author.num " - + "= (3.141592653589793)", + sql_query, expected_sql, ) self.assertEqual(params, ()) diff --git a/tests/unit/django_spanner/test_introspection.py b/tests/unit/django_spanner/test_introspection.py index c90288f3b3..b50cbc032b 100644 --- a/tests/unit/django_spanner/test_introspection.py +++ b/tests/unit/django_spanner/test_introspection.py @@ -11,6 +11,7 @@ from google.cloud.spanner_v1 import TypeCode from tests.unit.django_spanner.simple_test import SpannerSimpleTestClass from unittest import mock +from django_spanner import USING_DJANGO_3 class TestUtils(SpannerSimpleTestClass): @@ -49,9 +50,9 @@ def test_get_table_list(self): cursor = mock.MagicMock() def list_tables(*args, **kwargs): - return [["Table_1"], ["Table_2"]] + return [["Table_1", "t"], ["Table_2", "t"]] - cursor.list_tables = list_tables + cursor.run_sql_in_snapshot = list_tables table_list = db_introspection.get_table_list(cursor=cursor) self.assertEqual( table_list, @@ -86,31 +87,60 @@ def get_table_column_schema(*args, **kwargs): table_description = db_introspection.get_table_description( cursor=cursor, table_name="Table_1" ) - self.assertEqual( - table_description, - [ - FieldInfo( - name="name", - type_code=TypeCode.STRING, - display_size=None, - internal_size=10, - precision=None, - scale=None, - null_ok=False, - default=None, - ), - FieldInfo( - name="age", - type_code=TypeCode.INT64, - display_size=None, - internal_size=None, - precision=None, - scale=None, - null_ok=True, - default=None, - ), - ], - ) + if USING_DJANGO_3: + self.assertEqual( + table_description, + [ + FieldInfo( + name="name", + type_code=TypeCode.STRING, + display_size=None, + internal_size=10, + precision=None, + scale=None, + null_ok=False, + default=None, + collation=None, + ), + FieldInfo( + name="age", + type_code=TypeCode.INT64, + display_size=None, + internal_size=None, + precision=None, + scale=None, + null_ok=True, + default=None, + collation=None, + ), + ], + ) + else: + self.assertEqual( + table_description, + [ + FieldInfo( + name="name", + type_code=TypeCode.STRING, + display_size=None, + internal_size=10, + precision=None, + scale=None, + null_ok=False, + default=None, + ), + FieldInfo( + name="age", + type_code=TypeCode.INT64, + display_size=None, + internal_size=None, + precision=None, + scale=None, + null_ok=True, + default=None, + ), + ], + ) def test_get_primary_key_column(self): """ diff --git a/tests/unit/django_spanner/test_lookups.py b/tests/unit/django_spanner/test_lookups.py index 53604691cc..31f858b9a3 100644 --- a/tests/unit/django_spanner/test_lookups.py +++ b/tests/unit/django_spanner/test_lookups.py @@ -9,6 +9,7 @@ from tests.unit.django_spanner.simple_test import SpannerSimpleTestClass from decimal import Decimal from .models import Number, Author +from django_spanner import USING_DJANGO_3 class TestLookups(SpannerSimpleTestClass): @@ -56,11 +57,17 @@ def test_cast_param_to_float_with_no_params_query(self): qs1 = Number.objects.filter(item_id__exact=F("num")).values("num") compiler = SQLCompiler(qs1.query, self.connection, "default") sql_compiled, params = compiler.as_sql() - self.assertEqual( - sql_compiled, - "SELECT tests_number.num FROM tests_number WHERE " - + "tests_number.item_id = (tests_number.num)", - ) + if USING_DJANGO_3: + expected_sql = ( + "SELECT tests_number.num FROM tests_number WHERE " + + "tests_number.item_id = tests_number.num" + ) + else: + expected_sql = ( + "SELECT tests_number.num FROM tests_number WHERE " + + "tests_number.item_id = (tests_number.num)" + ) + self.assertEqual(sql_compiled, expected_sql) self.assertEqual(params, ()) def test_startswith_endswith_sql_query_with_startswith(self): @@ -107,13 +114,21 @@ def test_startswith_endswith_sql_query_with_bileteral_transform(self): compiler = SQLCompiler(qs1.query, self.connection, "default") sql_compiled, params = compiler.as_sql() - self.assertEqual( - sql_compiled, - "SELECT tests_author.name FROM tests_author WHERE " - + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " - + "REPLACE(REPLACE(REPLACE(CONCAT('^', (UPPER(%s))), " - + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))', - ) + if USING_DJANGO_3: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "REPLACE(REPLACE(REPLACE(CONCAT('^', UPPER(%s)), " + + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))' + ) + else: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "REPLACE(REPLACE(REPLACE(CONCAT('^', (UPPER(%s))), " + + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))' + ) + self.assertEqual(sql_compiled, expected_sql) self.assertEqual(params, ("abc",)) def test_startswith_endswith_case_insensitive_transform_sql_query(self): @@ -124,13 +139,21 @@ def test_startswith_endswith_case_insensitive_transform_sql_query(self): compiler = SQLCompiler(qs1.query, self.connection, "default") sql_compiled, params = compiler.as_sql() - self.assertEqual( - sql_compiled, - "SELECT tests_author.name FROM tests_author WHERE " - + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " - + "REPLACE(REPLACE(REPLACE(CONCAT('^(?i)', (UPPER(%s))), " - + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))', - ) + if USING_DJANGO_3: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "REPLACE(REPLACE(REPLACE(CONCAT('^(?i)', UPPER(%s)), " + + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))' + ) + else: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "REPLACE(REPLACE(REPLACE(CONCAT('^(?i)', (UPPER(%s))), " + + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))' + ) + self.assertEqual(sql_compiled, expected_sql) self.assertEqual(params, ("abc",)) def test_startswith_endswith_endswith_sql_query_with_transform(self): @@ -140,13 +163,21 @@ def test_startswith_endswith_endswith_sql_query_with_transform(self): compiler = SQLCompiler(qs1.query, self.connection, "default") sql_compiled, params = compiler.as_sql() - self.assertEqual( - sql_compiled, - "SELECT tests_author.name FROM tests_author WHERE " - + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " - + "REPLACE(REPLACE(REPLACE(CONCAT('', (UPPER(%s)), '$'), " - + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))', - ) + if USING_DJANGO_3: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "REPLACE(REPLACE(REPLACE(CONCAT('', UPPER(%s), '$'), " + + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))' + ) + else: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "REPLACE(REPLACE(REPLACE(CONCAT('', (UPPER(%s)), '$'), " + + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))' + ) + self.assertEqual(sql_compiled, expected_sql) self.assertEqual(params, ("abc",)) def test_regex_sql_query_case_sensitive(self): @@ -179,12 +210,19 @@ def test_regex_sql_query_case_sensitive_with_transform(self): compiler = SQLCompiler(qs1.query, self.connection, "default") sql_compiled, params = compiler.as_sql() - self.assertEqual( - sql_compiled, - "SELECT tests_author.num FROM tests_author WHERE " - + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " - + "(UPPER(%s)))", - ) + if USING_DJANGO_3: + expected_sql = ( + "SELECT tests_author.num FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "UPPER(%s))" + ) + else: + expected_sql = ( + "SELECT tests_author.num FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "(UPPER(%s)))" + ) + self.assertEqual(sql_compiled, expected_sql) self.assertEqual(params, ("abc",)) def test_regex_sql_query_case_insensitive_with_transform(self): @@ -193,12 +231,19 @@ def test_regex_sql_query_case_insensitive_with_transform(self): compiler = SQLCompiler(qs1.query, self.connection, "default") sql_compiled, params = compiler.as_sql() - self.assertEqual( - sql_compiled, - "SELECT tests_author.num FROM tests_author WHERE " - + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " - + "CONCAT('(?i)', (UPPER(%s))))", - ) + if USING_DJANGO_3: + expected_sql = ( + "SELECT tests_author.num FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "CONCAT('(?i)', UPPER(%s)))" + ) + else: + expected_sql = ( + "SELECT tests_author.num FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "CONCAT('(?i)', (UPPER(%s))))" + ) + self.assertEqual(sql_compiled, expected_sql) self.assertEqual(params, ("abc",)) def test_contains_sql_query_case_insensitive(self): @@ -232,13 +277,21 @@ def test_contains_sql_query_case_insensitive_transform(self): ) compiler = SQLCompiler(qs1.query, self.connection, "default") sql_compiled, params = compiler.as_sql() - self.assertEqual( - sql_compiled, - "SELECT tests_author.name FROM tests_author WHERE " - + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " - + "REPLACE(REPLACE(REPLACE(CONCAT('(?i)', (UPPER(%s))), " - + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))', - ) + if USING_DJANGO_3: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "REPLACE(REPLACE(REPLACE(CONCAT('(?i)', UPPER(%s)), " + + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))' + ) + else: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + "REPLACE(REPLACE(REPLACE(CONCAT('(?i)', (UPPER(%s))), " + + '"\\\\", "\\\\\\\\"), "%%", r"\\%%"), "_", r"\\_"))' + ) + self.assertEqual(sql_compiled, expected_sql) self.assertEqual(params, ("abc",)) def test_contains_sql_query_case_sensitive_transform(self): @@ -246,13 +299,21 @@ def test_contains_sql_query_case_sensitive_transform(self): qs1 = Author.objects.filter(name__upper__contains="abc").values("name") compiler = SQLCompiler(qs1.query, self.connection, "default") sql_compiled, params = compiler.as_sql() - self.assertEqual( - sql_compiled, - "SELECT tests_author.name FROM tests_author WHERE " - + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " - + 'REPLACE(REPLACE(REPLACE((UPPER(%s)), "\\\\", "\\\\\\\\"), ' - + '"%%", r"\\%%"), "_", r"\\_"))', - ) + if USING_DJANGO_3: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + 'REPLACE(REPLACE(REPLACE(UPPER(%s), "\\\\", "\\\\\\\\"), ' + + '"%%", r"\\%%"), "_", r"\\_"))' + ) + else: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(CAST(UPPER(tests_author.name) AS STRING), " + + 'REPLACE(REPLACE(REPLACE((UPPER(%s)), "\\\\", "\\\\\\\\"), ' + + '"%%", r"\\%%"), "_", r"\\_"))' + ) + self.assertEqual(sql_compiled, expected_sql) self.assertEqual(params, ("abc",)) def test_iexact_sql_query_case_insensitive(self): @@ -276,12 +337,19 @@ def test_iexact_sql_query_case_insensitive_function_transform(self): compiler = SQLCompiler(qs1.query, self.connection, "default") sql_compiled, params = compiler.as_sql() - self.assertEqual( - sql_compiled, - "SELECT tests_author.name FROM tests_author WHERE " - + "REGEXP_CONTAINS((UPPER(tests_author.last_name)), " - + "CONCAT('^(?i)', CAST(UPPER(tests_author.name) AS STRING), '$'))", - ) + if USING_DJANGO_3: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(UPPER(tests_author.last_name), " + + "CONCAT('^(?i)', CAST(UPPER(tests_author.name) AS STRING), '$'))" + ) + else: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS((UPPER(tests_author.last_name)), " + + "CONCAT('^(?i)', CAST(UPPER(tests_author.name) AS STRING), '$'))" + ) + self.assertEqual(sql_compiled, expected_sql) self.assertEqual(params, ()) def test_iexact_sql_query_case_insensitive_value_match(self): @@ -290,10 +358,17 @@ def test_iexact_sql_query_case_insensitive_value_match(self): compiler = SQLCompiler(qs1.query, self.connection, "default") sql_compiled, params = compiler.as_sql() - self.assertEqual( - sql_compiled, - "SELECT tests_author.name FROM tests_author WHERE " - + "REGEXP_CONTAINS((UPPER(CONCAT('^(?i)', " - + "CAST(UPPER(tests_author.name) AS STRING), '$'))), %s)", - ) + if USING_DJANGO_3: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS(UPPER(CONCAT('^(?i)', " + + "CAST(UPPER(tests_author.name) AS STRING), '$')), %s)" + ) + else: + expected_sql = ( + "SELECT tests_author.name FROM tests_author WHERE " + + "REGEXP_CONTAINS((UPPER(CONCAT('^(?i)', " + + "CAST(UPPER(tests_author.name) AS STRING), '$'))), %s)" + ) + self.assertEqual(sql_compiled, expected_sql) self.assertEqual(params, ("abc",)) diff --git a/tests/unit/django_spanner/test_utils.py b/tests/unit/django_spanner/test_utils.py index e4d50861d0..422adf6c8a 100644 --- a/tests/unit/django_spanner/test_utils.py +++ b/tests/unit/django_spanner/test_utils.py @@ -15,6 +15,8 @@ class TestUtils(SpannerSimpleTestClass): SQL_WITH_WHERE = "Select 1 from Table WHERE 1=1" SQL_WITHOUT_WHERE = "Select 1 from Table" + # Only active LTS django versions (2.2.*, 3.2.*) are supported by this library right now. + SUPPORTED_DJANGO_VERSIONS = [(2, 2), (3, 2)] def test_check_django_compatability_match(self): """ @@ -22,16 +24,16 @@ def test_check_django_compatability_match(self): """ django_spanner.__version__ = "2.2" django.VERSION = (2, 2, 19, "alpha", 0) - check_django_compatability() + check_django_compatability(self.SUPPORTED_DJANGO_VERSIONS) def test_check_django_compatability_mismatch(self): """ Checks django compatibility mismatch. """ django_spanner.__version__ = "2.2" - django.VERSION = (3, 2, 19, "alpha", 0) + django.VERSION = (3, 1, 19, "alpha", 0) with self.assertRaises(ImproperlyConfigured): - check_django_compatability() + check_django_compatability(self.SUPPORTED_DJANGO_VERSIONS) def test_add_dummy_where_with_where_present_and_not_added(self): """