diff --git a/datacube/index/abstract.py b/datacube/index/abstract.py index 87b671cfe..f7cc2c65d 100644 --- a/datacube/index/abstract.py +++ b/datacube/index/abstract.py @@ -21,7 +21,7 @@ from datacube.config import LocalConfig from datacube.index.exceptions import TransactionException from datacube.index.fields import Field -from datacube.model import Dataset, MetadataType, Range +from datacube.model import Dataset, MetadataType, Range, Not from datacube.model import Product from datacube.utils import cached_property, jsonify_document, read_documents, InvalidDocException from datacube.utils.changes import AllowPolicy, Change, Offset, DocumentMismatchError, check_doc_unchanged @@ -374,7 +374,7 @@ def get_all_docs(self) -> Iterable[Mapping[str, Any]]: yield mdt.definition -QueryField = Union[str, float, int, Range, datetime.datetime] +QueryField = Union[str, float, int, Range, datetime.datetime, Not] QueryDict = Mapping[str, QueryField] @@ -688,7 +688,7 @@ def search(self, **query: QueryField) -> Iterator[Product]: @abstractmethod def search_robust(self, **query: QueryField - ) -> Iterable[Tuple[Product, Mapping[str, QueryField]]]: + ) -> Iterable[Tuple[Product, QueryDict]]: """ Return dataset types that match match-able fields and dict of remaining un-matchable fields. @@ -698,7 +698,7 @@ def search_robust(self, @abstractmethod def search_by_metadata(self, - metadata: Mapping[str, QueryField] + metadata: QueryDict ) -> Iterable[Dataset]: """ Perform a search using arbitrary metadata, returning results as Product objects. @@ -1115,7 +1115,7 @@ def restore_location(self, @abstractmethod def search_by_metadata(self, - metadata: Mapping[str, QueryField] + metadata: QueryDict ) -> Iterable[Dataset]: """ Perform a search using arbitrary metadata, returning results as Dataset objects. @@ -1129,7 +1129,7 @@ def search_by_metadata(self, @abstractmethod def search(self, limit: Optional[int] = None, - source_filter: Optional[Mapping[str, QueryField]] = None, + source_filter: Optional[QueryDict] = None, **query: QueryField) -> Iterable[Dataset]: """ Perform a search, returning results as Dataset objects. diff --git a/datacube/index/fields.py b/datacube/index/fields.py index c67a1115d..2ee64acec 100644 --- a/datacube/index/fields.py +++ b/datacube/index/fields.py @@ -10,7 +10,7 @@ from dateutil.tz import tz from typing import List -from datacube.model import Range +from datacube.model import Range, Not from datacube.model.fields import Expression, Field __all__ = ['Field', @@ -36,6 +36,16 @@ def evaluate(self, ctx): return any(expr.evaluate(ctx) for expr in self.exprs) +class NotExpression(Expression): + def __init__(self, expr): + super(NotExpression, self).__init__() + self.expr = expr + self.field = expr.field + + def evaluate(self, ctx): + return not self.expr.evaluate(ctx) + + def as_expression(field: Field, value) -> Expression: """ Convert a single field/value to expression, following the "simple" conventions. @@ -44,6 +54,8 @@ def as_expression(field: Field, value) -> Expression: return field.between(value.begin, value.end) elif isinstance(value, list): return OrExpression(*(as_expression(field, val) for val in value)) + elif isinstance(value, Not): + return NotExpression(as_expression(field, value.value)) # Treat a date (day) as a time range. elif isinstance(value, date) and not isinstance(value, datetime): return as_expression( diff --git a/datacube/model/__init__.py b/datacube/model/__init__.py index 1880b3345..3c9b4ff65 100644 --- a/datacube/model/__init__.py +++ b/datacube/model/__init__.py @@ -21,7 +21,7 @@ schema_validated, DocReader from datacube.index.eo3 import is_doc_eo3 from .fields import Field, get_dataset_fields -from ._base import Range, ranges_overlap # noqa: F401 +from ._base import Range, ranges_overlap, Not # noqa: F401 from .eo3 import validate_eo3_compatible_type from deprecat import deprecat diff --git a/datacube/model/_base.py b/datacube/model/_base.py index 103765866..d1681e8fe 100644 --- a/datacube/model/_base.py +++ b/datacube/model/_base.py @@ -18,3 +18,6 @@ def ranges_overlap(ra: Range, rb: Range) -> bool: if ra.begin <= rb.begin: return ra.end > rb.begin return rb.end > ra.begin + + +Not = namedtuple('Not', 'value') diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst index b083c38a3..320d5ace0 100644 --- a/docs/about/whats_new.rst +++ b/docs/about/whats_new.rst @@ -18,6 +18,7 @@ v1.8.next - Fix broken codecov github action. (:pull:`1554`) - Throw error if ``time`` dimension is provided as an int or float to Query construction instead of assuming it to be seconds since epoch (:pull:`1561`) +- Add generic NOT operator and for ODC queries and ``Not`` type wrapper (:pull:`1563`) v1.8.17 (8th November 2023) =========================== diff --git a/integration_tests/index/test_config_docs.py b/integration_tests/index/test_config_docs.py index a5483dc1c..918c5623d 100644 --- a/integration_tests/index/test_config_docs.py +++ b/integration_tests/index/test_config_docs.py @@ -15,7 +15,7 @@ from datacube.index import Index from datacube.index.abstract import default_metadata_type_docs from datacube.model import MetadataType, DatasetType -from datacube.model import Range, Dataset +from datacube.model import Range, Not, Dataset from datacube.utils import changes from datacube.utils.documents import documents_equal from datacube.testutils import sanitise_doc @@ -447,7 +447,7 @@ def test_filter_types_by_fields(index, wo_eo3_product): assert len(res) == 0 -def test_filter_types_by_search(index, wo_eo3_product): +def test_filter_types_by_search(index, wo_eo3_product, ls8_eo3_product): """ :type ls5_telem_type: datacube.model.DatasetType :type index: datacube.index.Index @@ -456,7 +456,7 @@ def test_filter_types_by_search(index, wo_eo3_product): # No arguments, return all. res = list(index.products.search()) - assert res == [wo_eo3_product] + assert res == [ls8_eo3_product, wo_eo3_product] # Matching fields res = list(index.products.search( @@ -491,6 +491,12 @@ def test_filter_types_by_search(index, wo_eo3_product): )) assert res == [wo_eo3_product] + # Not expression test + res = list(index.products.search( + product_family=Not("wo"), + )) + assert res == [ls8_eo3_product] + # Mismatching fields res = list(index.products.search( product_family='spam',