Merge pull request #1452 from opendatacube/cli_maturity_model_2

Improve archive_less_mature compatibility with odc-tools

datacube/index/abstract.py

@@ -16,6 +16,7 @@
                     NamedTuple, Optional,
                     Tuple, Union, Sequence)
 from uuid import UUID
+from datetime import timedelta
 
 from datacube.config import LocalConfig
 from datacube.index.exceptions import TransactionException
@@ -904,10 +905,27 @@ def archive_less_mature(self, ds: Dataset) -> None:
 
         :param Dataset ds: dataset to search
         """
+        less_mature = self.find_less_mature(ds)
+        less_mature_ids = map(lambda x: x.id, less_mature)
+
+        self.archive(less_mature_ids)
+        for lm_ds in less_mature_ids:
+            _LOG.info(f"Archived less mature dataset: {lm_ds}")
+
+    def find_less_mature(self, ds: Dataset) -> Iterable[Dataset]:
+        """
+        Find less mature versions of a dataset
+
+        :param Dataset ds: Dataset to search
+        :return: Iterable of less mature datasets
+        """
         less_mature = []
+        # 'expand' the date range by a millisecond to give a bit more leniency in datetime comparison
+        expanded_time_range = Range(ds.metadata.time.begin - timedelta(milliseconds=1),
+                                    ds.metadata.time.end + timedelta(milliseconds=1))
         dupes = self.search(product=ds.product.name,
                             region_code=ds.metadata.region_code,
-                            time=ds.metadata.time)
+                            time=expanded_time_range)
         for dupe in dupes:
             if dupe.id == ds.id:
                 continue
@@ -924,10 +942,8 @@ def archive_less_mature(self, ds: Dataset) -> None:
                     f"A more mature version of dataset {ds.id} already exists, with id: "
                     f"{dupe.id} and maturity: {dupe.metadata.dataset_maturity}"
                 )
-            less_mature.append(dupe.id)
-        self.archive(less_mature)
-        for lm_ds in less_mature:
-            _LOG.info(f"Archived less mature dataset: {lm_ds}")
+            less_mature.append(dupe)
+        return less_mature
 
     @abstractmethod
     def restore(self, ids: Iterable[DSID]) -> None:

datacube/index/fields.py

@@ -38,7 +38,7 @@ def evaluate(self, ctx):
 
 def as_expression(field: Field, value) -> Expression:
     """
-    Convert a single field/value to expression, following the "simple" convensions.
+    Convert a single field/value to expression, following the "simple" conventions.
     """
     if isinstance(value, Range):
         return field.between(value.begin, value.end)

docs/about/whats_new.rst

@@ -18,7 +18,8 @@ v1.8.next
 - Pass Y and Y Scale factors through to rasterio.warp.reproject, to eliminate projection bug affecting
   non-square Areas Of Interest (See `Issue #1448`_) (:pull:`1450`)
 .. _`Issue #1448`: https://github.com/opendatacube/datacube-core/issues/1448
-- Add `archive_less_mature` option to `datacube dataset add` and `datacube dataset update` (:pull:`1451`)
+- Add `archive_less_mature` option to `datacube dataset add` and `datacube dataset update` (:pull:`1451`, :pull:`1452`)
+- Allow for +-1ms leniency in finding other maturity versions of a dataset (:pull:`1452`)
 
 
 v1.8.12 (7th March 2023)

integration_tests/conftest.py

@@ -182,6 +182,11 @@ def extended_eo3_metadata_type_doc():
     return get_eo3_test_data_doc("eo3_landsat_ard.odc-type.yaml")
 
 
+@pytest.fixture
+def eo3_sentinel_metadata_type_doc():
+    return get_eo3_test_data_doc("eo3_sentinel_ard.odc-type.yaml")
+
+
 @pytest.fixture
 def extended_eo3_product_doc():
     return get_eo3_test_data_doc("ard_ls8.odc-product.yaml")
@@ -197,6 +202,11 @@ def africa_s2_product_doc():
     return get_eo3_test_data_doc("s2_africa_product.yaml")
 
 
+@pytest.fixture
+def s2_ard_product_doc():
+    return get_eo3_test_data_doc("ga_s2am_ard_3.odc-product.yaml")
+
+
 @pytest.fixture
 def final_dataset_doc():
     return (
@@ -215,6 +225,24 @@ def nrt_dataset_doc():
     )
 
 
+@pytest.fixture
+def ga_s2am_ard_3_interim_doc():
+    return (
+        get_eo3_test_data_doc("ga_s2am_ard_3_interim.yaml"),
+        's3://dea-public-data/baseline/ga_s2am_ard_3/53/JNN/2021/07/24_interim'
+        '20230222T235626/ga_s2am_ard_3-2-1_53JNN_2021-07-24_interim.odc-metadata.yaml'
+    )
+
+
+@pytest.fixture
+def ga_s2am_ard_3_final_doc():
+    return (
+        get_eo3_test_data_doc("ga_s2am_ard_3_final.yaml"),
+        's3://dea-public-data/baseline/ga_s2am_ard_3/53/JNN/2021/07/24'
+        '20210724T023436/ga_s2am_ard_3-2-1_53JNN_2021-07-24_final.odc-metadata.yaml'
+    )
+
+
 def doc_to_ds(index, product_name, ds_doc, ds_path):
     from datacube.index.hl import Doc2Dataset
     resolver = Doc2Dataset(index, products=[product_name], verify_lineage=False)
@@ -224,13 +252,28 @@ def doc_to_ds(index, product_name, ds_doc, ds_path):
     return index.datasets.get(ds.id)
 
 
+def doc_to_ds_no_add(index, product_name, ds_doc, ds_path):
+    from datacube.index.hl import Doc2Dataset
+    resolver = Doc2Dataset(index, products=[product_name], verify_lineage=False)
+    ds, err = resolver(ds_doc, ds_path)
+    assert err is None and ds is not None
+    return ds
+
+
 @pytest.fixture
 def extended_eo3_metadata_type(index, extended_eo3_metadata_type_doc):
     return index.metadata_types.add(
         index.metadata_types.from_doc(extended_eo3_metadata_type_doc)
     )
 
 
+@pytest.fixture
+def eo3_sentinel_metadata_type(index, eo3_sentinel_metadata_type_doc):
+    return index.metadata_types.add(
+        index.metadata_types.from_doc(eo3_sentinel_metadata_type_doc)
+    )
+
+
 @pytest.fixture
 def ls8_eo3_product(index, extended_eo3_metadata_type, extended_eo3_product_doc):
     return index.products.add_document(extended_eo3_product_doc)
@@ -246,6 +289,11 @@ def africa_s2_eo3_product(index, africa_s2_product_doc):
     return index.products.add_document(africa_s2_product_doc)
 
 
+@pytest.fixture
+def ga_s2am_ard_3_product(index, eo3_sentinel_metadata_type, s2_ard_product_doc):
+    return index.products.add_document(s2_ard_product_doc)
+
+
 @pytest.fixture
 def eo3_products(index, extended_eo3_metadata_type,
                  ls8_eo3_product, wo_eo3_product,
@@ -301,24 +349,34 @@ def africa_eo3_dataset(index, africa_s2_eo3_product, eo3_africa_dataset_doc):
 
 @pytest.fixture
 def nrt_dataset(index, extended_eo3_metadata_type, ls8_eo3_product, nrt_dataset_doc):
-    ds_doc = nrt_dataset_doc[0]
-    ds_path = nrt_dataset_doc[1]
-    from datacube.index.hl import Doc2Dataset
-    resolver = Doc2Dataset(index, products=[ls8_eo3_product.name], verify_lineage=False)
-    ds, err = resolver(ds_doc, ds_path)
-    assert err is None and ds is not None
-    return ds
+    return doc_to_ds_no_add(
+        index,
+        ls8_eo3_product.name,
+        *nrt_dataset_doc)
 
 
 @pytest.fixture
 def final_dataset(index, extended_eo3_metadata_type, ls8_eo3_product, final_dataset_doc):
-    ds_doc = final_dataset_doc[0]
-    ds_path = final_dataset_doc[1]
-    from datacube.index.hl import Doc2Dataset
-    resolver = Doc2Dataset(index, products=[ls8_eo3_product.name], verify_lineage=False)
-    ds, err = resolver(ds_doc, ds_path)
-    assert err is None and ds is not None
-    return ds
+    return doc_to_ds_no_add(
+        index,
+        ls8_eo3_product.name,
+        *final_dataset_doc)
+
+
+@pytest.fixture
+def ga_s2am_ard3_final(index, eo3_sentinel_metadata_type, ga_s2am_ard_3_product, ga_s2am_ard_3_final_doc):
+    return doc_to_ds_no_add(
+        index,
+        ga_s2am_ard_3_product.name,
+        *ga_s2am_ard_3_final_doc)
+
+
+@pytest.fixture
+def ga_s2am_ard3_interim(index, eo3_sentinel_metadata_type, ga_s2am_ard_3_product, ga_s2am_ard_3_interim_doc):
+    return doc_to_ds_no_add(
+        index,
+        ga_s2am_ard_3_product.name,
+        *ga_s2am_ard_3_interim_doc)
 
 
 @pytest.fixture