From ed62385846a879d14a929c4542114b73b5bc051e Mon Sep 17 00:00:00 2001
From: Chris Wilcox <crwilcox@google.com>
Date: Tue, 31 Mar 2020 15:44:11 -0700
Subject: [PATCH 1/8] test: add tests for large offsets in datastore

---
 tests/system/test_system.py              | 60 ++++++++++++++++++++++++
 tests/system/utils/populate_datastore.py | 57 ++++++++++++++++++++++
 2 files changed, 117 insertions(+)

diff --git a/tests/system/test_system.py b/tests/system/test_system.py
index 482b0b80..29f89cee 100644
--- a/tests/system/test_system.py
+++ b/tests/system/test_system.py
@@ -14,6 +14,7 @@
 
 import datetime
 import os
+import string
 import unittest
 
 import requests
@@ -429,6 +430,9 @@ def test_query_paginate_with_offset(self):
         self.assertEqual(entities[1]["name"], "Jon Snow")
         self.assertEqual(entities[2]["name"], "Arya")
 
+    
+
+
     def test_query_paginate_with_start_cursor(self):
         page_query = self._base_query()
         page_query.order = "appearances"
@@ -464,6 +468,62 @@ def test_query_distinct_on(self):
         self.assertEqual(entities[0]["name"], "Catelyn")
         self.assertEqual(entities[1]["name"], "Arya")
 
+class TestDatastoreQueryOffsets(TestDatastore):
+    TOTAL_OBJECTS = 1500
+    NAMESPACE = "LargeCharacterEntity"
+    KIND = "LargeCharacter"
+
+    @classmethod
+    def setUpClass(cls):
+        cls.CLIENT = clone_client(Config.CLIENT)
+        # Remove the namespace from the cloned client, since these
+        # query tests rely on the entities to be already stored
+        cls.CLIENT.namespace = cls.NAMESPACE
+
+        # Populating the datastore if necessary.
+        populate_datastore.add_large_character_entities(client=cls.CLIENT)
+
+    @classmethod
+    def tearDownClass(cls):
+        # In the emulator, destroy the query entities.
+        if os.getenv(GCD_DATASET) is not None:
+            # Use the client for this test instead of the global.
+            clear_datastore.remove_all_entities(client=cls.CLIENT)
+
+    def _base_query(self):
+        # Use the client for this test instead of the global.
+        return self.CLIENT.query(
+            kind=self.KIND,
+            namespace=self.NAMESPACE
+        )
+
+    def _verify(self, limit, offset, expected):
+        # Query used for all tests
+        page_query = self._base_query()
+        page_query.add_filter("family", "=", "Stark")
+        page_query.add_filter("alive",  "=", False)
+
+        iterator = page_query.fetch(limit=limit, offset=offset)
+        entities = [e for e in iterator]
+        if len(entities) != expected:
+            self.fail(f"{limit}, {offset}, {expected}. Returned: {len(entities)}")
+
+    def test_query_in_bounds_offsets(self):
+        # Verify that with no offset there are the correct # of results
+        self._verify(limit=None, offset=None, expected=self.TOTAL_OBJECTS)
+        
+        # Verify that with no limit there are results (offset provided)")
+        self._verify(limit=None, offset=900, expected=self.TOTAL_OBJECTS-900)    
+
+        # Offset beyond items larger Verify 200 items found")
+        self._verify(limit=200, offset=1100, expected=200)
+
+    def test_query_out_of_bounds_offsets(self):
+        # Offset within range, expect 50 despite larger limit")
+        self._verify(limit=100, offset=self.TOTAL_OBJECTS-50, expected=50)
+
+        # Offset beyond items larger Verify no items found")
+        self._verify(limit=200, offset=self.TOTAL_OBJECTS+1000, expected=0)
 
 class TestDatastoreTransaction(TestDatastore):
     def test_transaction_via_with_statement(self):
diff --git a/tests/system/utils/populate_datastore.py b/tests/system/utils/populate_datastore.py
index 2c266a8a..14565101 100644
--- a/tests/system/utils/populate_datastore.py
+++ b/tests/system/utils/populate_datastore.py
@@ -18,6 +18,7 @@
 from __future__ import print_function
 
 import os
+import string
 import sys
 import time
 import uuid
@@ -61,6 +62,62 @@ def print_func(message):
     if os.getenv("GOOGLE_CLOUD_NO_PRINT") != "true":
         print(message)
 
+def add_large_character_entities(client=None):
+    TOTAL_OBJECTS = 1500
+    NAMESPACE="LargeCharacterEntity"
+    KIND="LargeCharacter"
+    MAX_STRING = (string.ascii_lowercase * 58)[:1500]
+
+    client.namespace = NAMESPACE
+
+    # Query used for all tests
+    page_query = client.query(
+        kind=KIND,
+        namespace=NAMESPACE,
+    )
+
+    def put_objects(count):
+        breakpoint()
+        remaining = count
+        current=0
+        
+        # Can only do 500 operations in a transaction with an overall
+        # size limit.
+        ENTITIES_TO_BATCH = 25
+        while current < count:
+            start = current
+            end = min(current + ENTITIES_TO_BATCH, count)
+            with client.transaction() as xact:
+                # The name/ID for the new entity
+                for i in range(start,end):
+                    name = f'character{i:05d}'
+                    # The Cloud Datastore key for the new entity
+                    task_key = client.key(KIND, name)
+
+                    # Prepares the new entity
+                    task = datastore.Entity(key=task_key)
+                    task['name'] = f"{i:05d}"
+                    task['family'] = 'Stark'
+                    task['alive'] = False
+
+                    for i in string.ascii_lowercase:
+                        task[f'space-{i}'] = MAX_STRING
+                        
+                    # Saves the entity
+                    xact.put(task)         
+            current += ENTITIES_TO_BATCH                     
+
+    # Ensure we have 1500 entities for tests. If not, clean up type and add
+    # new entities equal to TOTAL_OBJECTS
+    all_entities = [e for e in page_query.fetch()]
+    if len(all_entities) != TOTAL_OBJECTS:
+        # Cleanup Collection if not an exact match
+        while all_entities:
+            entities = all_entities[:500]
+            all_entities = all_entities[500:]
+            client.delete_multi([e.key for e in entities])
+        # Put objects
+        put_objects(TOTAL_OBJECTS)
 
 def add_characters(client=None):
     if client is None:

From cb77a748f438f8b36863964c565cbc0151cac8a0 Mon Sep 17 00:00:00 2001
From: Chris Wilcox <crwilcox@google.com>
Date: Mon, 6 Apr 2020 14:36:03 -0700
Subject: [PATCH 2/8] chore: remove unneeded blank lines

---
 tests/system/test_system.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/system/test_system.py b/tests/system/test_system.py
index 29f89cee..b4ca3049 100644
--- a/tests/system/test_system.py
+++ b/tests/system/test_system.py
@@ -430,9 +430,6 @@ def test_query_paginate_with_offset(self):
         self.assertEqual(entities[1]["name"], "Jon Snow")
         self.assertEqual(entities[2]["name"], "Arya")
 
-    
-
-
     def test_query_paginate_with_start_cursor(self):
         page_query = self._base_query()
         page_query.order = "appearances"

From 5a2a638852a046926b2acc2d551abc46df471f31 Mon Sep 17 00:00:00 2001
From: Chris Wilcox <crwilcox@google.com>
Date: Mon, 6 Apr 2020 15:24:03 -0700
Subject: [PATCH 3/8] fix: ensure we exhaust the query offset before returning
 entities

---
 google/cloud/datastore/query.py          | 13 ++++++++++++-
 tests/system/test_system.py              |  2 +-
 tests/system/utils/populate_datastore.py |  3 +--
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/google/cloud/datastore/query.py b/google/cloud/datastore/query.py
index 4b3daa66..29a4c121 100644
--- a/google/cloud/datastore/query.py
+++ b/google/cloud/datastore/query.py
@@ -498,7 +498,6 @@ def _process_query_results(self, response_pb):
         :raises ValueError: If ``more_results`` is an unexpected value.
         """
         self._skipped_results = response_pb.batch.skipped_results
-
         if response_pb.batch.more_results == _NO_MORE_RESULTS:
             self.next_page_token = None
         else:
@@ -540,6 +539,18 @@ def _next_page(self):
         response_pb = self.client._datastore_api.run_query(
             self._query.project, partition_id, read_options, query=query_pb
         )
+
+        while response_pb.batch.more_results == _NOT_FINISHED and response_pb.batch.skipped_results < query_pb.offset:
+            # We haven't finished processing. A likely reason is we haven't
+            # skipped all of the results yet. Don't return any results.
+            # Instead, rerun query, adjusting offsets. Datastore doesn't process
+            # more than 1000 skipped results in a query.
+            query_pb.start_cursor = response_pb.batch.skipped_cursor
+            query_pb.offset -= response_pb.batch.skipped_results
+            response_pb = self.client._datastore_api.run_query(
+                self._query.project, partition_id, read_options, query=query_pb
+            )
+
         entity_pbs = self._process_query_results(response_pb)
         return page_iterator.Page(self, entity_pbs, self.item_to_value)
 
diff --git a/tests/system/test_system.py b/tests/system/test_system.py
index b4ca3049..5a4b5d52 100644
--- a/tests/system/test_system.py
+++ b/tests/system/test_system.py
@@ -466,7 +466,7 @@ def test_query_distinct_on(self):
         self.assertEqual(entities[1]["name"], "Arya")
 
 class TestDatastoreQueryOffsets(TestDatastore):
-    TOTAL_OBJECTS = 1500
+    TOTAL_OBJECTS = 2500
     NAMESPACE = "LargeCharacterEntity"
     KIND = "LargeCharacter"
 
diff --git a/tests/system/utils/populate_datastore.py b/tests/system/utils/populate_datastore.py
index 14565101..31041a40 100644
--- a/tests/system/utils/populate_datastore.py
+++ b/tests/system/utils/populate_datastore.py
@@ -63,7 +63,7 @@ def print_func(message):
         print(message)
 
 def add_large_character_entities(client=None):
-    TOTAL_OBJECTS = 1500
+    TOTAL_OBJECTS = 2500
     NAMESPACE="LargeCharacterEntity"
     KIND="LargeCharacter"
     MAX_STRING = (string.ascii_lowercase * 58)[:1500]
@@ -77,7 +77,6 @@ def add_large_character_entities(client=None):
     )
 
     def put_objects(count):
-        breakpoint()
         remaining = count
         current=0
         

From 8829b9ab4f24da1ac1da83775075a3c56e0148b3 Mon Sep 17 00:00:00 2001
From: Chris Wilcox <crwilcox@google.com>
Date: Mon, 6 Apr 2020 15:24:47 -0700
Subject: [PATCH 4/8] test: split two tests that were failing before fix

---
 tests/system/test_system.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/system/test_system.py b/tests/system/test_system.py
index 5a4b5d52..f90e138b 100644
--- a/tests/system/test_system.py
+++ b/tests/system/test_system.py
@@ -515,10 +515,11 @@ def test_query_in_bounds_offsets(self):
         # Offset beyond items larger Verify 200 items found")
         self._verify(limit=200, offset=1100, expected=200)
 
-    def test_query_out_of_bounds_offsets(self):
+    def test_query_partially_out_of_bounds_offsets(self):
         # Offset within range, expect 50 despite larger limit")
         self._verify(limit=100, offset=self.TOTAL_OBJECTS-50, expected=50)
-
+    
+    def test_query_out_of_bounds_offsets(self):
         # Offset beyond items larger Verify no items found")
         self._verify(limit=200, offset=self.TOTAL_OBJECTS+1000, expected=0)
 

From ac895a72033b2e89c60eb5186e22fecb77af9bf3 Mon Sep 17 00:00:00 2001
From: Chris Wilcox <crwilcox@google.com>
Date: Mon, 6 Apr 2020 15:51:52 -0700
Subject: [PATCH 5/8] chore: blacken

---
 tests/system/test_system.py              | 43 ++++++++++----------
 tests/system/utils/populate_datastore.py | 51 ++++++++++++------------
 2 files changed, 46 insertions(+), 48 deletions(-)

diff --git a/tests/system/test_system.py b/tests/system/test_system.py
index f90e138b..8e0a7eb5 100644
--- a/tests/system/test_system.py
+++ b/tests/system/test_system.py
@@ -108,10 +108,10 @@ def setUpClass(cls):
 
     def _get_post(self, id_or_name=None, post_content=None):
         post_content = post_content or {
-            "title": u"How to make the perfect pizza in your grill",
-            "tags": [u"pizza", u"grill"],
+            "title": "How to make the perfect pizza in your grill",
+            "tags": ["pizza", "grill"],
             "publishedAt": datetime.datetime(2001, 1, 1, tzinfo=UTC),
-            "author": u"Silvano",
+            "author": "Silvano",
             "isDraft": False,
             "wordCount": 400,
             "rating": 5.0,
@@ -161,10 +161,10 @@ def test_save_multiple(self):
             self.case_entities_to_delete.append(entity1)
 
             second_post_content = {
-                "title": u"How to make the perfect homemade pasta",
-                "tags": [u"pasta", u"homemade"],
+                "title": "How to make the perfect homemade pasta",
+                "tags": ["pasta", "homemade"],
                 "publishedAt": datetime.datetime(2001, 1, 1),
-                "author": u"Silvano",
+                "author": "Silvano",
                 "isDraft": False,
                 "wordCount": 450,
                 "rating": 4.5,
@@ -193,7 +193,7 @@ def test_all_value_types(self):
         entity["truthy"] = True
         entity["float"] = 2.718281828
         entity["int"] = 3735928559
-        entity["words"] = u"foo"
+        entity["words"] = "foo"
         entity["blob"] = b"seekretz"
         entity_stored = datastore.Entity(key=key_stored)
         entity_stored["hi"] = "bye"
@@ -216,7 +216,7 @@ def test_save_key_self_reference(self):
         parent_key = Config.CLIENT.key("Residence", "NewYork")
         key = Config.CLIENT.key("Person", "name", parent=parent_key)
         entity = datastore.Entity(key=key)
-        entity["fullName"] = u"Full name"
+        entity["fullName"] = "Full name"
         entity["linkedTo"] = key  # Self reference.
 
         Config.CLIENT.put(entity)
@@ -465,6 +465,7 @@ def test_query_distinct_on(self):
         self.assertEqual(entities[0]["name"], "Catelyn")
         self.assertEqual(entities[1]["name"], "Arya")
 
+
 class TestDatastoreQueryOffsets(TestDatastore):
     TOTAL_OBJECTS = 2500
     NAMESPACE = "LargeCharacterEntity"
@@ -489,16 +490,13 @@ def tearDownClass(cls):
 
     def _base_query(self):
         # Use the client for this test instead of the global.
-        return self.CLIENT.query(
-            kind=self.KIND,
-            namespace=self.NAMESPACE
-        )
+        return self.CLIENT.query(kind=self.KIND, namespace=self.NAMESPACE)
 
     def _verify(self, limit, offset, expected):
         # Query used for all tests
         page_query = self._base_query()
         page_query.add_filter("family", "=", "Stark")
-        page_query.add_filter("alive",  "=", False)
+        page_query.add_filter("alive", "=", False)
 
         iterator = page_query.fetch(limit=limit, offset=offset)
         entities = [e for e in iterator]
@@ -508,25 +506,26 @@ def _verify(self, limit, offset, expected):
     def test_query_in_bounds_offsets(self):
         # Verify that with no offset there are the correct # of results
         self._verify(limit=None, offset=None, expected=self.TOTAL_OBJECTS)
-        
+
         # Verify that with no limit there are results (offset provided)")
-        self._verify(limit=None, offset=900, expected=self.TOTAL_OBJECTS-900)    
+        self._verify(limit=None, offset=900, expected=self.TOTAL_OBJECTS - 900)
 
         # Offset beyond items larger Verify 200 items found")
         self._verify(limit=200, offset=1100, expected=200)
 
     def test_query_partially_out_of_bounds_offsets(self):
         # Offset within range, expect 50 despite larger limit")
-        self._verify(limit=100, offset=self.TOTAL_OBJECTS-50, expected=50)
-    
+        self._verify(limit=100, offset=self.TOTAL_OBJECTS - 50, expected=50)
+
     def test_query_out_of_bounds_offsets(self):
         # Offset beyond items larger Verify no items found")
-        self._verify(limit=200, offset=self.TOTAL_OBJECTS+1000, expected=0)
+        self._verify(limit=200, offset=self.TOTAL_OBJECTS + 1000, expected=0)
+
 
 class TestDatastoreTransaction(TestDatastore):
     def test_transaction_via_with_statement(self):
         entity = datastore.Entity(key=Config.CLIENT.key("Company", "Google"))
-        entity["url"] = u"www.google.com"
+        entity["url"] = "www.google.com"
 
         with Config.CLIENT.transaction() as xact:
             result = Config.CLIENT.get(entity.key)
@@ -582,7 +581,7 @@ def test_failure_with_contention(self):
         # and updated outside it with a contentious value.
         key = local_client.key("BreakTxn", 1234)
         orig_entity = datastore.Entity(key=key)
-        orig_entity["foo"] = u"bar"
+        orig_entity["foo"] = "bar"
         local_client.put(orig_entity)
         self.case_entities_to_delete.append(orig_entity)
 
@@ -591,12 +590,12 @@ def test_failure_with_contention(self):
                 entity_in_txn = local_client.get(key)
 
                 # Update the original entity outside the transaction.
-                orig_entity[contention_prop_name] = u"outside"
+                orig_entity[contention_prop_name] = "outside"
                 Config.CLIENT.put(orig_entity)
 
                 # Try to update the entity which we already updated outside the
                 # transaction.
-                entity_in_txn[contention_prop_name] = u"inside"
+                entity_in_txn[contention_prop_name] = "inside"
                 txn.put(entity_in_txn)
 
     def test_empty_array_put(self):
diff --git a/tests/system/utils/populate_datastore.py b/tests/system/utils/populate_datastore.py
index 31041a40..326e9d11 100644
--- a/tests/system/utils/populate_datastore.py
+++ b/tests/system/utils/populate_datastore.py
@@ -42,19 +42,19 @@
     EDDARD + ("Character", "Jon Snow"),
 )
 CHARACTERS = (
-    {"name": u"Rickard", "family": u"Stark", "appearances": 0, "alive": False},
-    {"name": u"Eddard", "family": u"Stark", "appearances": 9, "alive": False},
+    {"name": "Rickard", "family": "Stark", "appearances": 0, "alive": False},
+    {"name": "Eddard", "family": "Stark", "appearances": 9, "alive": False},
     {
-        "name": u"Catelyn",
-        "family": [u"Stark", u"Tully"],
+        "name": "Catelyn",
+        "family": ["Stark", "Tully"],
         "appearances": 26,
         "alive": False,
     },
-    {"name": u"Arya", "family": u"Stark", "appearances": 33, "alive": True},
-    {"name": u"Sansa", "family": u"Stark", "appearances": 31, "alive": True},
-    {"name": u"Robb", "family": u"Stark", "appearances": 22, "alive": False},
-    {"name": u"Bran", "family": u"Stark", "appearances": 25, "alive": True},
-    {"name": u"Jon Snow", "family": u"Stark", "appearances": 32, "alive": True},
+    {"name": "Arya", "family": "Stark", "appearances": 33, "alive": True},
+    {"name": "Sansa", "family": "Stark", "appearances": 31, "alive": True},
+    {"name": "Robb", "family": "Stark", "appearances": 22, "alive": False},
+    {"name": "Bran", "family": "Stark", "appearances": 25, "alive": True},
+    {"name": "Jon Snow", "family": "Stark", "appearances": 32, "alive": True},
 )
 
 
@@ -62,24 +62,22 @@ def print_func(message):
     if os.getenv("GOOGLE_CLOUD_NO_PRINT") != "true":
         print(message)
 
+
 def add_large_character_entities(client=None):
     TOTAL_OBJECTS = 2500
-    NAMESPACE="LargeCharacterEntity"
-    KIND="LargeCharacter"
+    NAMESPACE = "LargeCharacterEntity"
+    KIND = "LargeCharacter"
     MAX_STRING = (string.ascii_lowercase * 58)[:1500]
 
     client.namespace = NAMESPACE
 
     # Query used for all tests
-    page_query = client.query(
-        kind=KIND,
-        namespace=NAMESPACE,
-    )
+    page_query = client.query(kind=KIND, namespace=NAMESPACE,)
 
     def put_objects(count):
         remaining = count
-        current=0
-        
+        current = 0
+
         # Can only do 500 operations in a transaction with an overall
         # size limit.
         ENTITIES_TO_BATCH = 25
@@ -88,23 +86,23 @@ def put_objects(count):
             end = min(current + ENTITIES_TO_BATCH, count)
             with client.transaction() as xact:
                 # The name/ID for the new entity
-                for i in range(start,end):
-                    name = f'character{i:05d}'
+                for i in range(start, end):
+                    name = f"character{i:05d}"
                     # The Cloud Datastore key for the new entity
                     task_key = client.key(KIND, name)
 
                     # Prepares the new entity
                     task = datastore.Entity(key=task_key)
-                    task['name'] = f"{i:05d}"
-                    task['family'] = 'Stark'
-                    task['alive'] = False
+                    task["name"] = f"{i:05d}"
+                    task["family"] = "Stark"
+                    task["alive"] = False
 
                     for i in string.ascii_lowercase:
-                        task[f'space-{i}'] = MAX_STRING
-                        
+                        task[f"space-{i}"] = MAX_STRING
+
                     # Saves the entity
-                    xact.put(task)         
-            current += ENTITIES_TO_BATCH                     
+                    xact.put(task)
+            current += ENTITIES_TO_BATCH
 
     # Ensure we have 1500 entities for tests. If not, clean up type and add
     # new entities equal to TOTAL_OBJECTS
@@ -118,6 +116,7 @@ def put_objects(count):
         # Put objects
         put_objects(TOTAL_OBJECTS)
 
+
 def add_characters(client=None):
     if client is None:
         # Get a client that uses the test dataset.

From 0e9eb664d6805a2a41a1f733c55717453c135073 Mon Sep 17 00:00:00 2001
From: Chris Wilcox <crwilcox@google.com>
Date: Mon, 6 Apr 2020 15:56:41 -0700
Subject: [PATCH 6/8] chore: blacken

---
 google/cloud/datastore/query.py          | 5 ++++-
 tests/system/utils/populate_datastore.py | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/google/cloud/datastore/query.py b/google/cloud/datastore/query.py
index 29a4c121..78a153cb 100644
--- a/google/cloud/datastore/query.py
+++ b/google/cloud/datastore/query.py
@@ -540,7 +540,10 @@ def _next_page(self):
             self._query.project, partition_id, read_options, query=query_pb
         )
 
-        while response_pb.batch.more_results == _NOT_FINISHED and response_pb.batch.skipped_results < query_pb.offset:
+        while (
+            response_pb.batch.more_results == _NOT_FINISHED
+            and response_pb.batch.skipped_results < query_pb.offset
+        ):
             # We haven't finished processing. A likely reason is we haven't
             # skipped all of the results yet. Don't return any results.
             # Instead, rerun query, adjusting offsets. Datastore doesn't process
diff --git a/tests/system/utils/populate_datastore.py b/tests/system/utils/populate_datastore.py
index 326e9d11..c6bc56ea 100644
--- a/tests/system/utils/populate_datastore.py
+++ b/tests/system/utils/populate_datastore.py
@@ -72,7 +72,7 @@ def add_large_character_entities(client=None):
     client.namespace = NAMESPACE
 
     # Query used for all tests
-    page_query = client.query(kind=KIND, namespace=NAMESPACE,)
+    page_query = client.query(kind=KIND, namespace=NAMESPACE)
 
     def put_objects(count):
         remaining = count

From 1e51a28cc3d646e3ca5ed31b10f447d0376f9a08 Mon Sep 17 00:00:00 2001
From: Chris Wilcox <crwilcox@google.com>
Date: Mon, 6 Apr 2020 16:46:07 -0700
Subject: [PATCH 7/8] chore: undo blackening of untouched code

---
 tests/system/test_system.py              | 27 ++++++++++++------------
 tests/system/utils/populate_datastore.py | 18 ++++++++--------
 2 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/tests/system/test_system.py b/tests/system/test_system.py
index 8e0a7eb5..5ee52f58 100644
--- a/tests/system/test_system.py
+++ b/tests/system/test_system.py
@@ -108,10 +108,10 @@ def setUpClass(cls):
 
     def _get_post(self, id_or_name=None, post_content=None):
         post_content = post_content or {
-            "title": "How to make the perfect pizza in your grill",
-            "tags": ["pizza", "grill"],
+            "title": u"How to make the perfect pizza in your grill",
+            "tags": [u"pizza", u"grill"],
             "publishedAt": datetime.datetime(2001, 1, 1, tzinfo=UTC),
-            "author": "Silvano",
+            "author": u"Silvano",
             "isDraft": False,
             "wordCount": 400,
             "rating": 5.0,
@@ -161,10 +161,10 @@ def test_save_multiple(self):
             self.case_entities_to_delete.append(entity1)
 
             second_post_content = {
-                "title": "How to make the perfect homemade pasta",
-                "tags": ["pasta", "homemade"],
+                "title": u"How to make the perfect homemade pasta",
+                "tags": [u"pasta", u"homemade"],
                 "publishedAt": datetime.datetime(2001, 1, 1),
-                "author": "Silvano",
+                "author": u"Silvano",
                 "isDraft": False,
                 "wordCount": 450,
                 "rating": 4.5,
@@ -193,7 +193,7 @@ def test_all_value_types(self):
         entity["truthy"] = True
         entity["float"] = 2.718281828
         entity["int"] = 3735928559
-        entity["words"] = "foo"
+        entity["words"] = u"foo"
         entity["blob"] = b"seekretz"
         entity_stored = datastore.Entity(key=key_stored)
         entity_stored["hi"] = "bye"
@@ -216,7 +216,7 @@ def test_save_key_self_reference(self):
         parent_key = Config.CLIENT.key("Residence", "NewYork")
         key = Config.CLIENT.key("Person", "name", parent=parent_key)
         entity = datastore.Entity(key=key)
-        entity["fullName"] = "Full name"
+        entity["fullName"] = u"Full name"
         entity["linkedTo"] = key  # Self reference.
 
         Config.CLIENT.put(entity)
@@ -476,7 +476,8 @@ def setUpClass(cls):
         cls.CLIENT = clone_client(Config.CLIENT)
         # Remove the namespace from the cloned client, since these
         # query tests rely on the entities to be already stored
-        cls.CLIENT.namespace = cls.NAMESPACE
+        # cls.CLIENT.namespace = cls.NAMESPACE
+        cls.CLIENT.namespace = None
 
         # Populating the datastore if necessary.
         populate_datastore.add_large_character_entities(client=cls.CLIENT)
@@ -525,7 +526,7 @@ def test_query_out_of_bounds_offsets(self):
 class TestDatastoreTransaction(TestDatastore):
     def test_transaction_via_with_statement(self):
         entity = datastore.Entity(key=Config.CLIENT.key("Company", "Google"))
-        entity["url"] = "www.google.com"
+        entity["url"] = u"www.google.com"
 
         with Config.CLIENT.transaction() as xact:
             result = Config.CLIENT.get(entity.key)
@@ -581,7 +582,7 @@ def test_failure_with_contention(self):
         # and updated outside it with a contentious value.
         key = local_client.key("BreakTxn", 1234)
         orig_entity = datastore.Entity(key=key)
-        orig_entity["foo"] = "bar"
+        orig_entity["foo"] = u"bar"
         local_client.put(orig_entity)
         self.case_entities_to_delete.append(orig_entity)
 
@@ -590,12 +591,12 @@ def test_failure_with_contention(self):
                 entity_in_txn = local_client.get(key)
 
                 # Update the original entity outside the transaction.
-                orig_entity[contention_prop_name] = "outside"
+                orig_entity[contention_prop_name] = u"outside"
                 Config.CLIENT.put(orig_entity)
 
                 # Try to update the entity which we already updated outside the
                 # transaction.
-                entity_in_txn[contention_prop_name] = "inside"
+                entity_in_txn[contention_prop_name] = u"inside"
                 txn.put(entity_in_txn)
 
     def test_empty_array_put(self):
diff --git a/tests/system/utils/populate_datastore.py b/tests/system/utils/populate_datastore.py
index c6bc56ea..b38caaf1 100644
--- a/tests/system/utils/populate_datastore.py
+++ b/tests/system/utils/populate_datastore.py
@@ -42,19 +42,19 @@
     EDDARD + ("Character", "Jon Snow"),
 )
 CHARACTERS = (
-    {"name": "Rickard", "family": "Stark", "appearances": 0, "alive": False},
-    {"name": "Eddard", "family": "Stark", "appearances": 9, "alive": False},
+    {"name": u"Rickard", "family": u"Stark", "appearances": 0, "alive": False},
+    {"name": u"Eddard", "family": u"Stark", "appearances": 9, "alive": False},
     {
-        "name": "Catelyn",
-        "family": ["Stark", "Tully"],
+        "name": u"Catelyn",
+        "family": [u"Stark", u"Tully"],
         "appearances": 26,
         "alive": False,
     },
-    {"name": "Arya", "family": "Stark", "appearances": 33, "alive": True},
-    {"name": "Sansa", "family": "Stark", "appearances": 31, "alive": True},
-    {"name": "Robb", "family": "Stark", "appearances": 22, "alive": False},
-    {"name": "Bran", "family": "Stark", "appearances": 25, "alive": True},
-    {"name": "Jon Snow", "family": "Stark", "appearances": 32, "alive": True},
+    {"name": u"Arya", "family": u"Stark", "appearances": 33, "alive": True},
+    {"name": u"Sansa", "family": u"Stark", "appearances": 31, "alive": True},
+    {"name": u"Robb", "family": u"Stark", "appearances": 22, "alive": False},
+    {"name": u"Bran", "family": u"Stark", "appearances": 25, "alive": True},
+    {"name": u"Jon Snow", "family": u"Stark", "appearances": 32, "alive": True},
 )
 
 

From ca0088ba3eefdbfc925dff2f9980af5a9fa6f93b Mon Sep 17 00:00:00 2001
From: Chris Wilcox <crwilcox@google.com>
Date: Mon, 6 Apr 2020 17:33:28 -0700
Subject: [PATCH 8/8] chore: remove usage of f-strings

---
 tests/system/test_system.py              | 3 +--
 tests/system/utils/populate_datastore.py | 6 +++---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/tests/system/test_system.py b/tests/system/test_system.py
index 5ee52f58..ef0de3a2 100644
--- a/tests/system/test_system.py
+++ b/tests/system/test_system.py
@@ -501,8 +501,7 @@ def _verify(self, limit, offset, expected):
 
         iterator = page_query.fetch(limit=limit, offset=offset)
         entities = [e for e in iterator]
-        if len(entities) != expected:
-            self.fail(f"{limit}, {offset}, {expected}. Returned: {len(entities)}")
+        self.assertEqual(len(entities), expected)
 
     def test_query_in_bounds_offsets(self):
         # Verify that with no offset there are the correct # of results
diff --git a/tests/system/utils/populate_datastore.py b/tests/system/utils/populate_datastore.py
index b38caaf1..e2baa5b3 100644
--- a/tests/system/utils/populate_datastore.py
+++ b/tests/system/utils/populate_datastore.py
@@ -87,18 +87,18 @@ def put_objects(count):
             with client.transaction() as xact:
                 # The name/ID for the new entity
                 for i in range(start, end):
-                    name = f"character{i:05d}"
+                    name = "character{0:05d}".format(i)
                     # The Cloud Datastore key for the new entity
                     task_key = client.key(KIND, name)
 
                     # Prepares the new entity
                     task = datastore.Entity(key=task_key)
-                    task["name"] = f"{i:05d}"
+                    task["name"] = "{0:05d}".format(i)
                     task["family"] = "Stark"
                     task["alive"] = False
 
                     for i in string.ascii_lowercase:
-                        task[f"space-{i}"] = MAX_STRING
+                        task["space-{}".format(i)] = MAX_STRING
 
                     # Saves the entity
                     xact.put(task)