Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(db_api): add an ability to set ReadOnly/ReadWrite connection mode #475

Merged
merged 37 commits into from Oct 5, 2021
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
4d7684e
feat(db_api): add an ability to set ReadOnly/ReadWrite connection mode
Aug 2, 2021
990fe3e
update the error message
Aug 2, 2021
2381c6d
update another error message
Aug 2, 2021
9cec921
don't check error messages with regexes
Aug 2, 2021
d6783b8
don't commit/rollback ReadOnly transactions
Aug 5, 2021
131a6fc
clear the transaction
Aug 5, 2021
e1a3db2
add read-only transactions data visibility test
Aug 5, 2021
4bd65e7
Apply suggestions from code review
Aug 5, 2021
4186609
add conditions for edge cases
Aug 6, 2021
0f3e716
Merge branch 'read_only_transactions' of https://github.com/q-logic/p…
Aug 6, 2021
8332d49
don't calc checksum for read-only transactions
Aug 6, 2021
61fc8ad
Merge branch 'master' into read_only_transactions
Aug 6, 2021
0be92af
use Snapshot for reads
Aug 10, 2021
3a72694
update docstrings
Aug 10, 2021
00887ff
Merge branch 'master' into read_only_transactions
Aug 10, 2021
b8eefed
Merge branch 'master' into read_only_transactions
Aug 11, 2021
de0e47e
use multi-use snapshots in !autocommit mode
Aug 11, 2021
9f1896a
return the read_only docstring back, erase excess unit test
Aug 11, 2021
4422ad5
Merge branch 'master' into read_only_transactions
larkee Aug 16, 2021
f11ea8c
erase excess ifs
Aug 16, 2021
9bafc76
Merge branch 'read_only_transactions' of https://github.com/q-logic/p…
Aug 16, 2021
783951f
Merge branch 'master' into read_only_transactions
Aug 20, 2021
d807a2d
add additional check into the snapshot_checkout() method
Aug 23, 2021
76e7caf
Merge branch 'master' into read_only_transactions
Aug 26, 2021
c61f212
add new style system test
Sep 13, 2021
9118987
Merge branch 'read_only_transactions' of https://github.com/q-logic/p…
Sep 13, 2021
70fc3ac
resolve conflict
Sep 13, 2021
22e5e73
don't use error message regexes
Sep 13, 2021
1cdccbe
erase excess import
Sep 13, 2021
9bb0ebc
Merge branch 'master' into read_only_transactions
Sep 14, 2021
ac8c4b2
refactor
Sep 14, 2021
ae9cd00
Merge branch 'main' into read_only_transactions
larkee Sep 16, 2021
6973748
add unit test to check that read-only transactions are not retried
Sep 16, 2021
5dab169
Merge branch 'read_only_transactions' of https://github.com/q-logic/p…
Sep 16, 2021
5761c30
Merge branch 'main' into read_only_transactions
skuruppu Sep 21, 2021
afeb585
Merge branch 'main' into read_only_transactions
skuruppu Oct 5, 2021
1ccb387
Merge branch 'main' into read_only_transactions
Oct 5, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
49 changes: 45 additions & 4 deletions google/cloud/spanner_dbapi/connection.py
Expand Up @@ -48,9 +48,14 @@ class Connection:

:type database: :class:`~google.cloud.spanner_v1.database.Database`
:param database: The database to which the connection is linked.

:type read_only: bool
:param read_only:
Flag to designate if the connection must use only ReadOnly
transaction type.
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved
"""

def __init__(self, instance, database):
def __init__(self, instance, database, read_only=False):
self._instance = instance
self._database = database
self._ddl_statements = []
Expand All @@ -67,6 +72,7 @@ def __init__(self, instance, database):
# this connection should be cleared on the
# connection close
self._own_pool = True
self._read_only = read_only

@property
def autocommit(self):
Expand Down Expand Up @@ -121,6 +127,31 @@ def instance(self):
"""
return self._instance

@property
def read_only(self):
"""Flag: the connection can be used only for database reads.

Returns:
bool:
True, if the connection intended to be used for
database reads only.
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved
"""
return self._read_only

@read_only.setter
def read_only(self, value):
"""`read_only` flag setter.

Args:
value (bool): True for ReadOnly mode, False for ReadWrite.
"""
if self.inside_transaction:
raise ValueError(
"Connection read/write mode can't be changed while a transaction is in progress. "
"Commit or rollback the current transaction and try again."
)
self._read_only = value
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved

def _session_checkout(self):
"""Get a Cloud Spanner session from the pool.

Expand Down Expand Up @@ -209,7 +240,7 @@ def transaction_checkout(self):
if not self.autocommit:
if not self.inside_transaction:
self._transaction = self._session_checkout().transaction()
self._transaction.begin()
self._transaction.begin(read_only=self._read_only)

return self._transaction

Expand Down Expand Up @@ -248,7 +279,12 @@ def commit(self):
self.run_prior_DDL_statements()
if self.inside_transaction:
try:
self._transaction.commit()
if not self.read_only:
self._transaction.commit()

self._session._transaction = None
self._transaction = None

self._release_session()
self._statements = []
except Aborted:
Expand All @@ -264,7 +300,12 @@ def rollback(self):
if self._autocommit:
warnings.warn(AUTOCOMMIT_MODE_WARNING, UserWarning, stacklevel=2)
elif self._transaction:
self._transaction.rollback()
if not self.read_only:
self._transaction.rollback()

self._session._transaction = None
self._transaction = None

self._release_session()
self._statements = []

Expand Down
12 changes: 10 additions & 2 deletions google/cloud/spanner_v1/transaction.py
Expand Up @@ -80,9 +80,14 @@ def _make_txn_selector(self):
self._check_state()
return TransactionSelector(id=self._transaction_id)

def begin(self):
def begin(self, read_only=False):
"""Begin a transaction on the database.

:type read_only: bool
:param read_only:
(Optional) If True, ReadOnly transaction type will be
begun, ReadWrite otherwise.
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved

:rtype: bytes
:returns: the ID for the newly-begun transaction.
:raises ValueError:
Expand All @@ -100,7 +105,10 @@ def begin(self):
database = self._session._database
api = database.spanner_api
metadata = _metadata_with_prefix(database.name)
txn_options = TransactionOptions(read_write=TransactionOptions.ReadWrite())
if read_only:
txn_options = TransactionOptions(read_only=TransactionOptions.ReadOnly())
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved
else:
txn_options = TransactionOptions(read_write=TransactionOptions.ReadWrite())
with trace_call("CloudSpanner.BeginTransaction", self._session):
response = api.begin_transaction(
session=self._session.name, options=txn_options, metadata=metadata
Expand Down
47 changes: 47 additions & 0 deletions tests/system/test_system_dbapi.py
Expand Up @@ -26,6 +26,7 @@
from google.cloud.spanner_v1.instance import Instance

from google.cloud.spanner_dbapi.connection import Connection
from google.cloud.spanner_dbapi.exceptions import ProgrammingError

from test_utils.retry import RetryErrors

Expand Down Expand Up @@ -426,6 +427,52 @@ def test_DDL_commit(self):
cur.execute("DROP TABLE Singers")
conn.commit()

def test_read_only(self):
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved
"""
Check that connection set to `read_only=True` uses
ReadOnly transactions.
"""
conn = Connection(Config.INSTANCE, self._db, read_only=True)
cur = conn.cursor()

with self.assertRaises(ProgrammingError):
cur.execute(
"""
UPDATE contacts
SET first_name = 'updated-first-name'
WHERE first_name = 'first-name'
"""
)

cur.execute("SELECT * FROM contacts")

conn.commit()

def test_read_only_data_visibility(self):
conn_rw = Connection(Config.INSTANCE, self._db)
conn_ro = Connection(Config.INSTANCE, self._db, read_only=True)

cur_rw = conn_rw.cursor()
cur_ro = conn_ro.cursor()

# start read-only transaction
cur_ro.execute("SELECT * FROM contacts")

cur_rw.execute(
"""INSERT INTO contacts (contact_id, first_name) VALUES (123, 'Inserted_while_read')"""
)
conn_rw.commit()

# try to read data inserted in parallel transaction
cur_ro.execute("SELECT * FROM contacts")
self.assertEqual(cur_ro.fetchall(), [])

# start new read-only transaction
conn_ro.commit()
cur_ro.execute("SELECT * FROM contacts")

self.assertEqual(cur_ro.fetchall(), [(123, "Inserted_while_read", None, None)])


def clear_table(transaction):
"""Clear the test table."""
Expand Down
44 changes: 42 additions & 2 deletions tests/unit/spanner_dbapi/test_connection.py
Expand Up @@ -39,14 +39,14 @@ def _get_client_info(self):

return ClientInfo(user_agent=USER_AGENT)

def _make_connection(self):
def _make_connection(self, **kwargs):
from google.cloud.spanner_dbapi import Connection
from google.cloud.spanner_v1.instance import Instance

# We don't need a real Client object to test the constructor
instance = Instance(INSTANCE, client=None)
database = instance.database(DATABASE)
return Connection(instance, database)
return Connection(instance, database, **kwargs)

@mock.patch("google.cloud.spanner_dbapi.connection.Connection.commit")
def test_autocommit_setter_transaction_not_started(self, mock_commit):
Expand Down Expand Up @@ -105,6 +105,46 @@ def test_property_instance(self):
self.assertIsInstance(connection.instance, Instance)
self.assertEqual(connection.instance, connection._instance)

def test_read_only_connection(self):
connection = self._make_connection(read_only=True)
self.assertTrue(connection.read_only)

connection._transaction = mock.Mock(committed=False, rolled_back=False)
with self.assertRaisesRegex(
ValueError,
"Connection read/write mode can't be changed while a transaction is in progress. "
"Commit or rollback the current transaction and try again.",
):
connection.read_only = False

connection._transaction = None
connection.read_only = False
self.assertFalse(connection.read_only)

def test_read_only_rollback_commit(self):
"""
Check that ReadOnly transactions are not committed
or rolled back, but sessions are still released.
"""
connection = self._make_connection(read_only=True)

transaction = mock.Mock(committed=False, rolled_back=False)
transaction.commit = mock.Mock()
transaction.rollback = mock.Mock()
connection._release_session = mock.Mock()

connection._transaction = transaction

connection.commit()
transaction.commit.assert_not_called()
connection._release_session.assert_called_once()

connection._release_session.reset_mock()

connection.rollback()
transaction.rollback.assert_not_called()
connection._release_session.assert_called_once()

@staticmethod
def _make_pool():
from google.cloud.spanner_v1.pool import AbstractSessionPool
Expand Down
25 changes: 25 additions & 0 deletions tests/unit/test_transaction.py
Expand Up @@ -193,6 +193,31 @@ def test_begin_ok(self):
"CloudSpanner.BeginTransaction", attributes=TestTransaction.BASE_ATTRIBUTES
)

def test_begin_read_only(self):
from google.cloud.spanner_v1 import Transaction as TransactionPB

transaction_pb = TransactionPB(id=self.TRANSACTION_ID)
database = _Database()
api = database.spanner_api = _FauxSpannerAPI(
_begin_transaction_response=transaction_pb
)
session = _Session(database)
transaction = self._make_one(session)

txn_id = transaction.begin(read_only=True)

self.assertEqual(txn_id, self.TRANSACTION_ID)
self.assertEqual(transaction._transaction_id, self.TRANSACTION_ID)

session_id, txn_options, metadata = api._begun
self.assertEqual(session_id, session.name)
self.assertTrue(type(txn_options).pb(txn_options).HasField("read_only"))
self.assertEqual(metadata, [("google-cloud-resource-prefix", database.name)])

self.assertSpanAttributes(
"CloudSpanner.BeginTransaction", attributes=TestTransaction.BASE_ATTRIBUTES
)

def test_rollback_not_begun(self):
session = _Session()
transaction = self._make_one(session)
Expand Down