Merge pull request #137 from octue/release/0.1.14

Release/0.1.14

.github/workflows/release.yml

@@ -1,4 +1,4 @@
-name: Publish on merge of release/x.y.z into main
+name: Release the package on merge of release/x.y.z into main
 
 # Only trigger when a pull request into main branch is closed.
 on:

docs/source/deploying_services.rst

@@ -15,7 +15,7 @@ Automated deployment with Octue means:
 
 All you need to enable automated deployments are the following files in your repository root:
 
-* A ``requirements.txt`` file that includes ``octue>=0.1.13`` and the rest of your service's dependencies
+* A ``requirements.txt`` file that includes ``octue>=0.1.14`` and the rest of your service's dependencies
 * A ``twine.json`` file
 * A ``deployment_configuration.json`` file (optional)
 

octue/deployment/google/Dockerfile

@@ -8,10 +8,6 @@ WORKDIR $PROJECT_ROOT
 
 RUN apt-get update -y && apt-get install -y --fix-missing build-essential && rm -rf /var/lib/apt/lists/*
 
-# This will cache bust if any of the requirements change.
-COPY requirements*.txt .
-COPY setup.* .
-
 COPY . .
 
 # Install requirements (supports requirements.txt, requirements-dev.txt, and setup.py; all will be run if all are present.)

octue/mixins/serialisable.py

@@ -7,29 +7,36 @@ class Serialisable:
     """Mixin class to make resources serialisable to JSON.
 
     Objects must have a `.logger` and a `.id` property
-
     """
 
     _SERIALISE_FIELDS = None
     _EXCLUDE_SERIALISE_FIELDS = ("logger",)
 
     def __init__(self, *args, **kwargs):
         """Constructor for serialisable mixin"""
-        # Ensure it passes construction arguments up the chain
         super().__init__(*args, **kwargs)
 
+    @classmethod
+    def deserialise(cls, serialised_object):
+        """Deserialise the given JSON-serialised object.
+
+        :param dict serialised_object:
+        :return any:
+        """
+        return cls(**serialised_object)
+
     def to_file(self, file_name, **kwargs):
-        """Write to a JSON file
+        """Write the object to a JSON file.
 
-        :parameter file_name:  file to write to, including relative or absolute path and .json extension
-        :type file_name: path-like
+        :parameter str file_name: file to write to, including relative or absolute path and .json extension
+        :return None:
         """
         self.logger.debug("Writing %s %s to file %s", self.__class__.__name__, self.id, file_name)
         with open(file_name, "w") as fp:
             fp.write(self.serialise(**kwargs, to_string=True))
 
     def serialise(self, to_string=False, **kwargs):
-        """Serialise into a primitive dict or JSON string
+        """Serialise into a primitive dict or JSON string.
 
         Serialises all non-private and non-protected attributes except for 'logger', unless the subclass has a
         `_serialise_fields` tuple of the attribute names to serialise. For example:

octue/resources/datafile.py

@@ -204,7 +204,7 @@ def to_cloud(self, project_name, bucket_name, path_in_bucket):
                 "hash_value": self.hash_value,
                 "cluster": self.cluster,
                 "sequence": self.sequence,
-                "tags": self.tags.serialise(to_string=False),
+                "tags": self.tags.serialise(),
             },
         )
 

octue/resources/dataset.py

@@ -10,6 +10,7 @@
 from octue.mixins import Hashable, Identifiable, Loggable, Pathable, Serialisable, Taggable
 from octue.resources.datafile import Datafile
 from octue.resources.filter_containers import FilterSet
+from octue.resources.tag import TagSet
 
 
 module_logger = logging.getLogger(__name__)
@@ -27,6 +28,7 @@ class Dataset(Taggable, Serialisable, Pathable, Loggable, Identifiable, Hashable
 
     _FILTERSET_ATTRIBUTE = "files"
     _ATTRIBUTES_TO_HASH = "files", "name", "tags"
+    _SERIALISE_FIELDS = "files", "name", "tags", "hash_value", "id", "path"
 
     def __init__(self, name=None, id=None, logger=None, path=None, path_from=None, tags=None, **kwargs):
         """Construct a Dataset"""
@@ -84,7 +86,7 @@ def from_cloud(cls, project_name, bucket_name, path_to_dataset_directory):
             name=serialised_dataset["name"],
             hash_value=serialised_dataset["hash_value"],
             path=storage.path.generate_gs_path(bucket_name, path_to_dataset_directory),
-            tags=json.loads(serialised_dataset["tags"]),
+            tags=TagSet(serialised_dataset["tags"]),
             files=datafiles,
         )
 
@@ -107,7 +109,6 @@ def to_cloud(self, project_name, bucket_name, output_directory):
 
         serialised_dataset = self.serialise()
         serialised_dataset["files"] = sorted(files)
-        del serialised_dataset["absolute_path"]
         del serialised_dataset["path"]
 
         GoogleCloudStorageClient(project_name=project_name).upload_from_string(

octue/resources/filter_containers.py

@@ -4,20 +4,9 @@
 def _filter(instance, filter_name=None, filter_value=None):
     """Returns a new instance containing only the Filterables to which the given filter criteria apply.
 
-    Say we want to filter by files whose extension equals "csv". We want to be able to do...
-
-    ds = DataSet(... initialise with csv files and other files)
-    Example of chaining:
-
-    ds.filter('files__extension_equals', 'csv')
-    is equvalent to
-    ds.files.filter(extension__equals', 'csv')
-    >>> FilterSet containing a set of datafiles
-
-
-    ds.filter('files__extension_equals', 'csv')
-    is equvalent to
-    ds.files.filter(extension__equals', 'csv')
+    :param str filter_name:
+    :param any filter_value:
+    :return octue.resources.filter_containers.FilterSet:
     """
     return instance.__class__((item for item in instance if item.satisfies(filter_name, filter_value)))
 

octue/resources/manifest.py

@@ -17,6 +17,7 @@ class Manifest(Pathable, Serialisable, Loggable, Identifiable, Hashable):
     (or leaving), a data service for an analysis at the configuration, input or output stage."""
 
     _ATTRIBUTES_TO_HASH = "datasets", "keys"
+    _SERIALISE_FIELDS = "datasets", "keys", "hash_value", "id", "name", "path"
 
     def __init__(self, id=None, logger=None, path=None, datasets=None, keys=None, **kwargs):
         super().__init__(id=id, logger=logger, path=path)
@@ -43,20 +44,6 @@ def __init__(self, id=None, logger=None, path=None, datasets=None, keys=None, **
         self._instantiate_datasets(datasets, key_list)
         vars(self).update(**kwargs)
 
-    @classmethod
-    def deserialise(cls, serialised_manifest, from_string=False):
-        """Deserialise a Manifest from a dictionary."""
-        if from_string:
-            serialised_manifest = json.loads(serialised_manifest)
-
-        return cls(
-            name=serialised_manifest["name"],
-            id=serialised_manifest["id"],
-            datasets=serialised_manifest["datasets"],
-            keys=serialised_manifest["keys"],
-            path=serialised_manifest["path"],
-        )
-
     @classmethod
     def from_cloud(cls, project_name, bucket_name, path_to_manifest_file):
         """Instantiate a Manifest from Google Cloud storage.
@@ -113,7 +100,6 @@ def to_cloud(self, project_name, bucket_name, path_to_manifest_file, store_datas
 
         serialised_manifest = self.serialise()
         serialised_manifest["datasets"] = sorted(datasets)
-        del serialised_manifest["absolute_path"]
         del serialised_manifest["path"]
 
         GoogleCloudStorageClient(project_name=project_name).upload_from_string(

octue/resources/tag.py

@@ -3,8 +3,9 @@
 from functools import lru_cache
 
 from octue.exceptions import InvalidTagException
-from octue.mixins import Filterable
-from octue.resources.filter_containers import FilterSet
+from octue.mixins import Filterable, Serialisable
+from octue.resources.filter_containers import FilterList, FilterSet
+from octue.utils.encoders import OctueJSONEncoder
 
 
 TAG_PATTERN = re.compile(r"^$|^[a-z0-9][a-z0-9:\-]*(?<![:-])$")
@@ -28,8 +29,11 @@ def name(self):
     @property
     @lru_cache(maxsize=1)
     def subtags(self):
-        """ Return the subtags of the tag as a new TagSet (e.g. TagSet({'a', 'b', 'c'}) for the Tag('a:b:c'). """
-        return TagSet({Tag(subtag_name) for subtag_name in (self.name.split(":"))})
+        """Return the subtags of the tag in order as a FilterList (e.g. FilterList(['a', 'b', 'c']) for Tag('a:b:c').
+
+        :return FilterList(Tag):
+        """
+        return FilterList(Tag(subtag_name) for subtag_name in self.name.split(":"))
 
     def __eq__(self, other):
         if isinstance(other, str):
@@ -85,7 +89,7 @@ def _clean(name):
         return cleaned_name
 
 
-class TagSet:
+class TagSet(Serialisable):
     """ Class to handle a set of tags as a string. """
 
     _FILTERSET_ATTRIBUTE = "tags"
@@ -114,10 +118,6 @@ def __init__(self, tags=None, *args, **kwargs):
                 "Tags must be expressed as a whitespace-delimited string or an iterable of strings or Tag instances."
             )
 
-    def __str__(self):
-        """ Serialise tags to a sorted list string. """
-        return self.serialise()
-
     def __eq__(self, other):
         """ Does this TagSet have the same tags as another TagSet? """
         if not isinstance(other, TagSet):
@@ -161,10 +161,35 @@ def any_tag_contains(self, value):
         """ Return True if any of the tags contains value. """
         return any(value in tag for tag in self)
 
-    def serialise(self, to_string=True):
-        """ Serialise tags to a sorted list string. """
-        serialised_tags = sorted(tag.name for tag in self)
+    def filter(self, filter_name=None, filter_value=None):
+        """Filter the tags with the given filter for the given value.
+
+        :param str filter_name:
+        :param any filter_value:
+        :return octue.resources.filter_containers.FilterSet:
+        """
+        return self.tags.filter(filter_name=filter_name, filter_value=filter_value)
+
+    def serialise(self, to_string=False, **kwargs):
+        """Serialise to a sorted list of tag names.
+
+        :param bool to_string:
+        :return list|str:
+        """
+        string = json.dumps(
+            sorted(tag.name for tag in self.tags), cls=OctueJSONEncoder, sort_keys=True, indent=4, **kwargs
+        )
 
         if to_string:
-            return str(serialised_tags)
-        return serialised_tags
+            return string
+
+        return json.loads(string)
+
+    @classmethod
+    def deserialise(cls, serialised_tagset):
+        """Deserialise from a sorted list of tag names.
+
+        :param list serialised_tagset:
+        :return TagSet:
+        """
+        return cls(tags=serialised_tagset)

octue/templates/template-child-services/elevation_service/requirements.txt

@@ -1 +1 @@
-octue==0.1.13
+octue==0.1.14

octue/templates/template-child-services/parent_service/requirements.txt

@@ -1 +1 @@
-octue==0.1.13
+octue==0.1.14

octue/templates/template-child-services/wind_speed_service/requirements.txt

@@ -1 +1 @@
-octue==0.1.13
+octue==0.1.14

octue/templates/template-python-fractal/requirements.txt

@@ -1,4 +1,4 @@
-octue==0.1.13
+octue==0.1.14
 
 
 # ----------- Some common libraries  -----------------------------------------------------------------------------------

octue/templates/template-using-manifests/data/input/manifest.json

@@ -7,14 +7,14 @@
     {
       "id": "7ead4669-8162-4f64-8cd5-4abe92509e17",
       "name": "meteorological mast dataset",
-      "tags": "met mast wind location:108346",
+      "tags": ["met", "mast", "wind", "location:108346"],
       "files": [
         {
           "path": "08DEC/High Res Meteorological Mast Data - 8 Dec_1.csv",
           "cluster": 0,
           "sequence": 0,
           "extension": "csv",
-          "tags": "timeseries",
+          "tags": ["timeseries"],
           "timestamp": 1605783547.0,
           "id": "acff07bc-7c19-4ed5-be6d-a6546eae8e86",
           "name": "High Res Meteorological Mast Data - 8 Dec_1.csv",
@@ -26,7 +26,7 @@
           "cluster": 0,
           "sequence": 1,
           "extension": "csv",
-          "tags": "timeseries",
+          "tags": ["timeseries"],
           "timestamp": 1605783547.0,
           "id": "bdff07bc-7c19-4ed5-be6d-a6546eae8e45",
           "name": "High Res Meteorological Mast Data - 8 Dec_2.csv",
@@ -38,7 +38,7 @@
           "cluster": 1,
           "sequence": 0,
           "extension": "dat",
-          "tags": "meta",
+          "tags": ["meta"],
           "timestamp": 1605783547.0,
           "id": "ceff07bc-7c19-4ed5-be6d-a6546eae8e86",
           "name": "meta - 8 Dec_1.da",

octue/templates/template-using-manifests/requirements.txt

@@ -1,4 +1,4 @@
-octue==0.1.13
+octue==0.1.14
 
 
 # ----------- Some common libraries  -----------------------------------------------------------------------------------

setup.py

@@ -17,7 +17,7 @@
 
 setup(
     name="octue",
-    version="0.1.13",  # Ensure all requirements files containing octue are updated, too (e.g. docs build).
+    version="0.1.14",  # Ensure all requirements files containing octue are updated, too (e.g. docs build).
     py_modules=["cli"],
     install_requires=[
         "click>=7.1.2",
@@ -28,7 +28,7 @@
         "google-cloud-storage>=1.35.1",
         "google-crc32c>=1.1.2",
         "gunicorn",
-        "twined>=0.0.17",
+        "twined>=0.0.18",
     ],
     url="https://www.github.com/octue/octue-sdk-python",
     license="MIT",

tests/resources/test_tag.py

@@ -1,12 +1,12 @@
-from octue.resources.filter_containers import FilterSet
+from octue.resources.filter_containers import FilterList, FilterSet
 from octue.resources.tag import Tag, TagSet
 from tests.base import BaseTestCase
 
 
 class TestTag(BaseTestCase):
     def test_subtags(self):
         """ Test that subtags are correctly parsed from tags. """
-        self.assertEqual(Tag("a:b:c").subtags, TagSet({Tag("a"), Tag("b"), Tag("c")}))
+        self.assertEqual(Tag("a:b:c").subtags, FilterList([Tag("a"), Tag("b"), Tag("c")]))
 
     def test_tag_comparison(self):
         """ Test that tags can be alphabetically compared. """
@@ -42,8 +42,8 @@ def test_starts_with(self):
 
     def test_subtags_starts_with(self):
         """ Test that the start of subtags can be checked. """
-        self.assertTrue(Tag("hello:world").subtags.any_tag_starts_with("w"))
-        self.assertFalse(Tag("hello:world").subtags.any_tag_starts_with("e"))
+        self.assertTrue(TagSet(Tag("hello:world").subtags).any_tag_starts_with("w"))
+        self.assertFalse(TagSet(Tag("hello:world").subtags).any_tag_starts_with("e"))
 
     def test_ends_with(self):
         """ Test that the end of a tag can be checked. """
@@ -52,8 +52,8 @@ def test_ends_with(self):
 
     def test_subtags_ends_with(self):
         """ Test that the end of subtags can be checked. """
-        self.assertTrue(Tag("hello:world").subtags.any_tag_ends_with("o"))
-        self.assertFalse(Tag("hello:world").subtags.any_tag_ends_with("e"))
+        self.assertTrue(TagSet(Tag("hello:world").subtags).any_tag_ends_with("o"))
+        self.assertFalse(TagSet(Tag("hello:world").subtags).any_tag_ends_with("e"))
 
 
 class TestTagSet(BaseTestCase):
@@ -174,18 +174,19 @@ def test_filter_chaining(self):
 
     def test_serialise(self):
         """ Ensure that TagSets are serialised to the string form of a list. """
-        self.assertEqual(self.TAG_SET.serialise(), "['a', 'b:c', 'd:e:f']")
+        self.assertEqual(self.TAG_SET.serialise(), ["a", "b:c", "d:e:f"])
 
     def test_serialise_orders_tags(self):
-        """ Ensure that TagSets are serialised to the string form of a list. """
+        """Ensure that TagSets serialise to a list."""
         tag_set = TagSet("z hello a c:no")
-        self.assertEqual(tag_set.serialise(), "['a', 'c:no', 'hello', 'z']")
+        self.assertEqual(tag_set.serialise(), ["a", "c:no", "hello", "z"])
 
-    def test_str_is_equivalent_to_serialise(self):
-        """ Test that calling `str` on a TagSet is equivalent to using the `serialise` method. """
-        tag_set = TagSet("z hello a c:no")
-        self.assertEqual(str(tag_set), tag_set.serialise())
+    def test_deserialise(self):
+        """Test that serialisation is reversible."""
+        serialised_tag_set = self.TAG_SET.serialise()
+        deserialised_tag_set = TagSet.deserialise(serialised_tag_set)
+        self.assertEqual(deserialised_tag_set, self.TAG_SET)
 
     def test_repr(self):
-        """ Test the representation of a TagSet appears as expected. """
+        """Test the representation of a TagSet appears as expected."""
         self.assertEqual(repr(self.TAG_SET), f"<TagSet({repr(self.TAG_SET.tags)})>")