Merge pull request #1561 from opendatacube/fix_time_int_bug

Refine _time_to_search_dims logic

datacube/api/query.py

@@ -14,7 +14,6 @@
 from typing import Optional, Union
 import pandas
 
-from dateutil import tz
 from pandas import to_datetime as pandas_to_datetime
 import numpy as np
 
@@ -58,7 +57,7 @@ def __init__(self, group_by_func, dimension, units, sort_key=None, group_key=Non
               'source_filter')
 
 
-class Query(object):
+class Query:
     def __init__(self, index=None, product=None, geopolygon=None, like=None, **search_terms):
         """Parses search terms in preparation for querying the Data Cube Index.
 
@@ -69,7 +68,7 @@ def __init__(self, index=None, product=None, geopolygon=None, like=None, **searc
         Use by accessing :attr:`search_terms`:
 
         >>> query.search_terms['time']  # doctest: +NORMALIZE_WHITESPACE
-        Range(begin=datetime.datetime(2001, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), \
+        Range(begin=datetime.datetime(2001, 1, 1, 0, 0, tzinfo=tzutc()), \
         end=datetime.datetime(2002, 1, 1, 23, 59, 59, 999999, tzinfo=tzutc()))
 
         By passing in an ``index``, the search parameters will be validated as existing on the ``product``.
@@ -127,9 +126,9 @@ def __init__(self, index=None, product=None, geopolygon=None, like=None, **searc
                 time_coord = like.coords.get('time')
                 if time_coord is not None:
                     self.search['time'] = _time_to_search_dims(
+                        # convert from np.datetime64 to datetime.datetime
                         (pandas_to_datetime(time_coord.values[0]).to_pydatetime(),
-                            pandas_to_datetime(time_coord.values[-1]).to_pydatetime()
-                            + datetime.timedelta(milliseconds=1))  # TODO: inclusive time searches
+                         pandas_to_datetime(time_coord.values[-1]).to_pydatetime())
                     )
 
     @property
@@ -304,27 +303,9 @@ def _values_to_search(**kwargs):
     return search
 
 
-def _to_datetime(t):
-    if isinstance(t, (float, int)):
-        t = datetime.datetime.fromtimestamp(t, tz=tz.tzutc())
-
-    if isinstance(t, tuple):
-        t = datetime.datetime(*t, tzinfo=tz.tzutc())
-    elif isinstance(t, str):
-        try:
-            t = datetime.datetime.strptime(t, "%Y-%m-%dT%H:%M:%S.%fZ")
-        except ValueError:
-            pass
-    elif isinstance(t, datetime.datetime):
-        return tz_aware(t)
-
-    return pandas_to_datetime(t, utc=True, infer_datetime_format=True).to_pydatetime()
-
-
 def _time_to_search_dims(time_range):
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", UserWarning)
-
         tr_start, tr_end = time_range, time_range
 
         if hasattr(time_range, '__iter__') and not isinstance(time_range, str):
@@ -334,24 +315,29 @@ def _time_to_search_dims(time_range):
 
             tr_start, tr_end = tmp[0], tmp[-1]
 
-        # Attempt conversion to isoformat
-        # allows pandas.Period to handle
-        # date and datetime objects
-        if hasattr(tr_start, 'isoformat'):
-            tr_start = tr_start.isoformat()
-        if hasattr(tr_end, 'isoformat'):
-            tr_end = tr_end.isoformat()
+        if isinstance(tr_start, (int, float)) or isinstance(tr_end, (int, float)):
+            raise TypeError("Time dimension must be provided as a datetime or a string")
 
         if tr_start is None:
-            tr_start = datetime.datetime.fromtimestamp(0)
-        start = _to_datetime(tr_start)
+            start = datetime.datetime.fromtimestamp(0)
+        elif not isinstance(tr_start, datetime.datetime):
+            # convert to datetime.datetime
+            if hasattr(tr_start, 'isoformat'):
+                tr_start = tr_start.isoformat()
+            start = pandas_to_datetime(tr_start).to_pydatetime()
+        else:
+            start = tr_start
+
         if tr_end is None:
             tr_end = datetime.datetime.now().strftime("%Y-%m-%d")
-        end = _to_datetime(pandas.Period(tr_end)
-                           .end_time
-                           .to_pydatetime())
+        # Attempt conversion to isoformat
+        # allows pandas.Period to handle datetime objects
+        if hasattr(tr_end, 'isoformat'):
+            tr_end = tr_end.isoformat()
+        # get end of period to ensure range is inclusive
+        end = pandas.Period(tr_end).end_time.to_pydatetime()
 
-        tr = Range(start, end)
+        tr = Range(tz_aware(start), tz_aware(end))
         if start == end:
             return tr[0]
 

docs/about/whats_new.rst

@@ -16,6 +16,8 @@ v1.8.next
 - Tweak ``list_products`` logic for getting crs and resolution values (:pull:`1535`)
 - Add new ODC Cheatsheet reference doc to Data Access & Analysis documentation page (:pull:`1543`)
 - Fix broken codecov github action. (:pull:`1554`)
+- Throw error if ``time`` dimension is provided as an int or float to Query construction
+  instead of assuming it to be seconds since epoch (:pull:`1561`)
 - Add generic NOT operator and for ODC queries and ``Not`` type wrapper (:pull:`1563`)
 
 v1.8.17 (8th November 2023)

tests/api/test_query.py

@@ -144,7 +144,7 @@ def format_test(start_out, end_out):
     ((datetime.date(2008, 1, 1), None),
      format_test('2008-01-01T00:00:00', datetime.datetime.now().strftime("%Y-%m-%dT23:59:59.999999"))),
     ((None, '2008'),
-     format_test(datetime.datetime.fromtimestamp(0).strftime("%Y-%m-%dT%H:%M:%S"), '2008-12-31T23:59:59.999999'))
+     format_test(datetime.datetime.fromtimestamp(0).strftime("%Y-%m-%dT%H:%M:%S"), '2008-12-31T23:59:59.999999')),
 ]
 
 
@@ -155,6 +155,11 @@ def test_time_handling(time_param, expected):
     assert query.search_terms['time'] == expected
 
 
+def test_time_handling_int():
+    with pytest.raises(TypeError):
+        Query(time=2008)
+
+
 def test_solar_day():
     _s = SimpleNamespace
     ds = _s(center_time=parse_time('1987-05-22 23:07:44.2270250Z'),