Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: timestamp precision in insert_rows #393

Merged
merged 3 commits into from Nov 23, 2020
Merged

fix: timestamp precision in insert_rows #393

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0e69c1e
fix: timestamp precision in insert_rows
HemangChothani Nov 19, 2020
b139c07
fix: remove floating point coversion and add datetime format
HemangChothani Nov 20, 2020
c675232
fix: add formatted string in unit tests
HemangChothani Nov 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 3 additions & 6 deletions google/cloud/bigquery/_helpers.py
View file
Open in desktop
Expand Up @@ -23,7 +23,7 @@
from google.cloud._helpers import UTC
from google.cloud._helpers import _date_from_iso8601_date
from google.cloud._helpers import _datetime_from_microseconds
from google.cloud._helpers import _microseconds_from_datetime
from google.cloud._helpers import _RFC3339_MICROS
from google.cloud._helpers import _RFC3339_NO_FRACTION
from google.cloud._helpers import _to_bytes

Expand Down Expand Up @@ -313,12 +313,9 @@ def _timestamp_to_json_parameter(value):


def _timestamp_to_json_row(value):
"""Coerce 'value' to an JSON-compatible representation.

This version returns floating-point seconds value used in row data.
"""
"""Coerce 'value' to an JSON-compatible representation."""
if isinstance(value, datetime.datetime):
value = _microseconds_from_datetime(value) * 1e-6
value = value.strftime(_RFC3339_MICROS)
return value


Expand Down
11 changes: 9 additions & 2 deletions tests/unit/test__helpers.py
View file
Open in desktop
Expand Up @@ -728,10 +728,17 @@ def test_w_string(self):
self.assertEqual(self._call_fut(ZULU), ZULU)

def test_w_datetime(self):
from google.cloud._helpers import _microseconds_from_datetime
from google.cloud._helpers import _RFC3339_MICROS

when = datetime.datetime(2016, 12, 20, 15, 58, 27, 339328)
self.assertEqual(self._call_fut(when), _microseconds_from_datetime(when) / 1e6)
self.assertEqual(self._call_fut(when), when.strftime(_RFC3339_MICROS))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please update these tests to compare against the actual string produced. I trust that when.strftime(_RFC3339_MICROS) is doing the right thing, but I'd be much more comfortable seeing the actual formatted string here and in the other unit tests.


def test_w_datetime_w_utc_zone(self):
from google.cloud._helpers import _RFC3339_MICROS
from google.cloud._helpers import UTC

when = datetime.datetime(2020, 11, 17, 1, 6, 52, 353795, tzinfo=UTC)
self.assertEqual(self._call_fut(when), when.strftime(_RFC3339_MICROS))


class Test_datetime_to_json(unittest.TestCase):
Expand Down
25 changes: 17 additions & 8 deletions tests/unit/test_client.py
View file
Open in desktop
Expand Up @@ -5804,7 +5804,7 @@ def test_insert_rows_w_schema(self):
import datetime
from google.cloud._helpers import UTC
from google.cloud._helpers import _datetime_to_rfc3339
from google.cloud._helpers import _microseconds_from_datetime
from google.cloud._helpers import _RFC3339_MICROS
from google.cloud.bigquery.schema import SchemaField

WHEN_TS = 1437767599.006
Expand Down Expand Up @@ -5834,7 +5834,7 @@ def _row_data(row):
result = {"full_name": row[0], "age": str(row[1])}
joined = row[2]
if isinstance(joined, datetime.datetime):
joined = _microseconds_from_datetime(joined) * 1e-6
joined = joined.strftime(_RFC3339_MICROS)
if joined is not None:
result["joined"] = joined
return result
Expand Down Expand Up @@ -5864,7 +5864,7 @@ def test_insert_rows_w_list_of_dictionaries(self):
import datetime
from google.cloud._helpers import UTC
from google.cloud._helpers import _datetime_to_rfc3339
from google.cloud._helpers import _microseconds_from_datetime
from google.cloud._helpers import _RFC3339_MICROS
from google.cloud.bigquery.schema import SchemaField
from google.cloud.bigquery.table import Table

Expand Down Expand Up @@ -5910,7 +5910,7 @@ def _row_data(row):
row = copy.deepcopy(row)
del row["joined"]
elif isinstance(joined, datetime.datetime):
row["joined"] = _microseconds_from_datetime(joined) * 1e-6
row["joined"] = joined.strftime(_RFC3339_MICROS)
row["age"] = str(row["age"])
return row

Expand Down Expand Up @@ -6052,6 +6052,7 @@ def _row_data(row):
)

def test_insert_rows_w_repeated_fields(self):
from google.cloud._helpers import _RFC3339_MICROS
from google.cloud.bigquery.schema import SchemaField
from google.cloud.bigquery.table import Table

Expand Down Expand Up @@ -6109,16 +6110,24 @@ def test_insert_rows_w_repeated_fields(self):
{
"score": "12",
"times": [
1543665600.0, # 2018-12-01 12:00 UTC
1543669200.0, # 2018-12-01 13:00 UTC
datetime.datetime(
2018, 12, 1, 12, 0, 0, tzinfo=pytz.utc
).strftime(_RFC3339_MICROS),
datetime.datetime(
2018, 12, 1, 13, 0, 0, tzinfo=pytz.utc
).strftime(_RFC3339_MICROS),
],
"distances": [1.25, 2.5],
},
{
"score": "13",
"times": [
1543752000.0, # 2018-12-02 12:00 UTC
1543755600.0, # 2018-12-02 13:00 UTC
datetime.datetime(
2018, 12, 2, 12, 0, 0, tzinfo=pytz.utc
).strftime(_RFC3339_MICROS),
datetime.datetime(
2018, 12, 2, 13, 0, 0, tzinfo=pytz.utc
).strftime(_RFC3339_MICROS),
],
"distances": [-1.25, -2.5],
},
Expand Down