From ba2b37df016ac0ebfcf177e25f00adf24244eb1c Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Wed, 13 Mar 2024 05:18:49 +0000 Subject: [PATCH 1/4] add NotType and NotExpression --- datacube/index/abstract.py | 12 ++++++------ datacube/index/fields.py | 14 +++++++++++++- datacube/model/__init__.py | 2 +- datacube/model/_base.py | 3 +++ integration_tests/index/test_config_docs.py | 12 +++++++++--- 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/datacube/index/abstract.py b/datacube/index/abstract.py index 87b671cfe..3654eae0c 100644 --- a/datacube/index/abstract.py +++ b/datacube/index/abstract.py @@ -21,7 +21,7 @@ from datacube.config import LocalConfig from datacube.index.exceptions import TransactionException from datacube.index.fields import Field -from datacube.model import Dataset, MetadataType, Range +from datacube.model import Dataset, MetadataType, Range, NotType from datacube.model import Product from datacube.utils import cached_property, jsonify_document, read_documents, InvalidDocException from datacube.utils.changes import AllowPolicy, Change, Offset, DocumentMismatchError, check_doc_unchanged @@ -374,7 +374,7 @@ def get_all_docs(self) -> Iterable[Mapping[str, Any]]: yield mdt.definition -QueryField = Union[str, float, int, Range, datetime.datetime] +QueryField = Union[str, float, int, Range, datetime.datetime, NotType] QueryDict = Mapping[str, QueryField] @@ -688,7 +688,7 @@ def search(self, **query: QueryField) -> Iterator[Product]: @abstractmethod def search_robust(self, **query: QueryField - ) -> Iterable[Tuple[Product, Mapping[str, QueryField]]]: + ) -> Iterable[Tuple[Product, QueryDict]]: """ Return dataset types that match match-able fields and dict of remaining un-matchable fields. @@ -698,7 +698,7 @@ def search_robust(self, @abstractmethod def search_by_metadata(self, - metadata: Mapping[str, QueryField] + metadata: QueryDict ) -> Iterable[Dataset]: """ Perform a search using arbitrary metadata, returning results as Product objects. @@ -1115,7 +1115,7 @@ def restore_location(self, @abstractmethod def search_by_metadata(self, - metadata: Mapping[str, QueryField] + metadata: QueryDict ) -> Iterable[Dataset]: """ Perform a search using arbitrary metadata, returning results as Dataset objects. @@ -1129,7 +1129,7 @@ def search_by_metadata(self, @abstractmethod def search(self, limit: Optional[int] = None, - source_filter: Optional[Mapping[str, QueryField]] = None, + source_filter: Optional[QueryDict] = None, **query: QueryField) -> Iterable[Dataset]: """ Perform a search, returning results as Dataset objects. diff --git a/datacube/index/fields.py b/datacube/index/fields.py index c67a1115d..557edf4e9 100644 --- a/datacube/index/fields.py +++ b/datacube/index/fields.py @@ -10,7 +10,7 @@ from dateutil.tz import tz from typing import List -from datacube.model import Range +from datacube.model import Range, NotType from datacube.model.fields import Expression, Field __all__ = ['Field', @@ -36,6 +36,16 @@ def evaluate(self, ctx): return any(expr.evaluate(ctx) for expr in self.exprs) +class NotExpression(Expression): + def __init__(self, expr): + super(NotExpression, self).__init__() + self.expr = expr + self.field = expr.field + + def evaluate(self, ctx): + return not self.expr.evaluate(ctx) + + def as_expression(field: Field, value) -> Expression: """ Convert a single field/value to expression, following the "simple" conventions. @@ -44,6 +54,8 @@ def as_expression(field: Field, value) -> Expression: return field.between(value.begin, value.end) elif isinstance(value, list): return OrExpression(*(as_expression(field, val) for val in value)) + elif isinstance(value, NotType): + return NotExpression(as_expression(field, value.value)) # Treat a date (day) as a time range. elif isinstance(value, date) and not isinstance(value, datetime): return as_expression( diff --git a/datacube/model/__init__.py b/datacube/model/__init__.py index 1880b3345..14a346a47 100644 --- a/datacube/model/__init__.py +++ b/datacube/model/__init__.py @@ -21,7 +21,7 @@ schema_validated, DocReader from datacube.index.eo3 import is_doc_eo3 from .fields import Field, get_dataset_fields -from ._base import Range, ranges_overlap # noqa: F401 +from ._base import Range, ranges_overlap, NotType # noqa: F401 from .eo3 import validate_eo3_compatible_type from deprecat import deprecat diff --git a/datacube/model/_base.py b/datacube/model/_base.py index 103765866..9f5ca5f48 100644 --- a/datacube/model/_base.py +++ b/datacube/model/_base.py @@ -18,3 +18,6 @@ def ranges_overlap(ra: Range, rb: Range) -> bool: if ra.begin <= rb.begin: return ra.end > rb.begin return rb.end > ra.begin + + +NotType = namedtuple('NotType', 'value') diff --git a/integration_tests/index/test_config_docs.py b/integration_tests/index/test_config_docs.py index a5483dc1c..a4913e461 100644 --- a/integration_tests/index/test_config_docs.py +++ b/integration_tests/index/test_config_docs.py @@ -15,7 +15,7 @@ from datacube.index import Index from datacube.index.abstract import default_metadata_type_docs from datacube.model import MetadataType, DatasetType -from datacube.model import Range, Dataset +from datacube.model import Range, NotType, Dataset from datacube.utils import changes from datacube.utils.documents import documents_equal from datacube.testutils import sanitise_doc @@ -447,7 +447,7 @@ def test_filter_types_by_fields(index, wo_eo3_product): assert len(res) == 0 -def test_filter_types_by_search(index, wo_eo3_product): +def test_filter_types_by_search(index, wo_eo3_product, ls8_eo3_product): """ :type ls5_telem_type: datacube.model.DatasetType :type index: datacube.index.Index @@ -456,7 +456,7 @@ def test_filter_types_by_search(index, wo_eo3_product): # No arguments, return all. res = list(index.products.search()) - assert res == [wo_eo3_product] + assert res == [ls8_eo3_product, wo_eo3_product] # Matching fields res = list(index.products.search( @@ -491,6 +491,12 @@ def test_filter_types_by_search(index, wo_eo3_product): )) assert res == [wo_eo3_product] + # Not expression test + res = list(index.products.search( + product_family=NotType("wo"), + )) + assert res == [ls8_eo3_product] + # Mismatching fields res = list(index.products.search( product_family='spam', From b70ee5c90d2892a287363ca39c9ad987496e73e9 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Wed, 13 Mar 2024 23:59:52 +0000 Subject: [PATCH 2/4] update whats_new --- docs/about/whats_new.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index f8ecdc76a..ed57ea129 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -16,6 +16,7 @@ v1.8.next - Tweak ``list_products`` logic for getting crs and resolution values (:pull:`1535`) - Add new ODC Cheatsheet reference doc to Data Access & Analysis documentation page (:pull:`1543`) - Fix broken codecov github action. (:pull:`1554`) +- Add generic NOT operator and for ODC queries and ``NotType`` wrapper (:pull:`1563`) v1.8.17 (8th November 2023) =========================== From ae4f2f0ad17aa8bb14860130545fa5fee96caf4b Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Thu, 14 Mar 2024 00:14:21 +0000 Subject: [PATCH 3/4] update wordlist --- wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/wordlist.txt b/wordlist.txt index 12e238e1a..d8abe43b5 100644 --- a/wordlist.txt +++ b/wordlist.txt @@ -314,6 +314,7 @@ nodata NodeJS nosignatures NotImplementedError +NotType np NPM npm From 0b550ad6b4aa6bba357e0d8dcc8fb66b5f907702 Mon Sep 17 00:00:00 2001 From: Ariana Barzinpour Date: Thu, 14 Mar 2024 02:08:44 +0000 Subject: [PATCH 4/4] rename NotType to Not --- datacube/index/abstract.py | 4 ++-- datacube/index/fields.py | 4 ++-- datacube/model/__init__.py | 2 +- datacube/model/_base.py | 2 +- docs/about/whats_new.rst | 2 +- integration_tests/index/test_config_docs.py | 4 ++-- wordlist.txt | 1 - 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/datacube/index/abstract.py b/datacube/index/abstract.py index 3654eae0c..f7cc2c65d 100644 --- a/datacube/index/abstract.py +++ b/datacube/index/abstract.py @@ -21,7 +21,7 @@ from datacube.config import LocalConfig from datacube.index.exceptions import TransactionException from datacube.index.fields import Field -from datacube.model import Dataset, MetadataType, Range, NotType +from datacube.model import Dataset, MetadataType, Range, Not from datacube.model import Product from datacube.utils import cached_property, jsonify_document, read_documents, InvalidDocException from datacube.utils.changes import AllowPolicy, Change, Offset, DocumentMismatchError, check_doc_unchanged @@ -374,7 +374,7 @@ def get_all_docs(self) -> Iterable[Mapping[str, Any]]: yield mdt.definition -QueryField = Union[str, float, int, Range, datetime.datetime, NotType] +QueryField = Union[str, float, int, Range, datetime.datetime, Not] QueryDict = Mapping[str, QueryField] diff --git a/datacube/index/fields.py b/datacube/index/fields.py index 557edf4e9..2ee64acec 100644 --- a/datacube/index/fields.py +++ b/datacube/index/fields.py @@ -10,7 +10,7 @@ from dateutil.tz import tz from typing import List -from datacube.model import Range, NotType +from datacube.model import Range, Not from datacube.model.fields import Expression, Field __all__ = ['Field', @@ -54,7 +54,7 @@ def as_expression(field: Field, value) -> Expression: return field.between(value.begin, value.end) elif isinstance(value, list): return OrExpression(*(as_expression(field, val) for val in value)) - elif isinstance(value, NotType): + elif isinstance(value, Not): return NotExpression(as_expression(field, value.value)) # Treat a date (day) as a time range. elif isinstance(value, date) and not isinstance(value, datetime): diff --git a/datacube/model/__init__.py b/datacube/model/__init__.py index 14a346a47..3c9b4ff65 100644 --- a/datacube/model/__init__.py +++ b/datacube/model/__init__.py @@ -21,7 +21,7 @@ schema_validated, DocReader from datacube.index.eo3 import is_doc_eo3 from .fields import Field, get_dataset_fields -from ._base import Range, ranges_overlap, NotType # noqa: F401 +from ._base import Range, ranges_overlap, Not # noqa: F401 from .eo3 import validate_eo3_compatible_type from deprecat import deprecat diff --git a/datacube/model/_base.py b/datacube/model/_base.py index 9f5ca5f48..d1681e8fe 100644 --- a/datacube/model/_base.py +++ b/datacube/model/_base.py @@ -20,4 +20,4 @@ def ranges_overlap(ra: Range, rb: Range) -> bool: return rb.end > ra.begin -NotType = namedtuple('NotType', 'value') +Not = namedtuple('Not', 'value') diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index ed57ea129..0242881c2 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -16,7 +16,7 @@ v1.8.next - Tweak ``list_products`` logic for getting crs and resolution values (:pull:`1535`) - Add new ODC Cheatsheet reference doc to Data Access & Analysis documentation page (:pull:`1543`) - Fix broken codecov github action. (:pull:`1554`) -- Add generic NOT operator and for ODC queries and ``NotType`` wrapper (:pull:`1563`) +- Add generic NOT operator and for ODC queries and ``Not`` type wrapper (:pull:`1563`) v1.8.17 (8th November 2023) =========================== diff --git a/integration_tests/index/test_config_docs.py b/integration_tests/index/test_config_docs.py index a4913e461..918c5623d 100644 --- a/integration_tests/index/test_config_docs.py +++ b/integration_tests/index/test_config_docs.py @@ -15,7 +15,7 @@ from datacube.index import Index from datacube.index.abstract import default_metadata_type_docs from datacube.model import MetadataType, DatasetType -from datacube.model import Range, NotType, Dataset +from datacube.model import Range, Not, Dataset from datacube.utils import changes from datacube.utils.documents import documents_equal from datacube.testutils import sanitise_doc @@ -493,7 +493,7 @@ def test_filter_types_by_search(index, wo_eo3_product, ls8_eo3_product): # Not expression test res = list(index.products.search( - product_family=NotType("wo"), + product_family=Not("wo"), )) assert res == [ls8_eo3_product] diff --git a/wordlist.txt b/wordlist.txt index d8abe43b5..12e238e1a 100644 --- a/wordlist.txt +++ b/wordlist.txt @@ -314,7 +314,6 @@ nodata NodeJS nosignatures NotImplementedError -NotType np NPM npm